






import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


A__     :      int             =			logging.getLogger(__name__)
@dataclass
class 				_UpperCAmelCase :

				"""simple docstring"""




				lowercase__    							=	field(
				    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}     )
				lowercase__    							=	field(
				    default=A__		,metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}     )
				lowercase__    							=	field(
				    default=A__		,metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}     )
				lowercase__    							=	field(default=A__		,metadata={"""help""": """Set this flag to use fast tokenization."""}     )
				# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
				# or just modify its tokenizer_config.json.
				lowercase__    							=	field(
				    default=A__		,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}		,)






def   a      (						):






				'''simple docstring'''
				lowercase__       			=      HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)    )

				if len(sys.argv    ) == 2 and sys.argv[1].endswith('''.json'''    ):
								# If we pass only one argument to the script and it's the path to a json file,
								# let's parse it to get our arguments.
								lowercase__			,						lowercase__			,						lowercase__       			=      parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]    )    )
				else:
								lowercase__			,						lowercase__			,						lowercase__       			=      parser.parse_args_into_dataclasses()

				if (
				    os.path.exists(training_args.output_dir    )
				    and os.listdir(training_args.output_dir    )
				    and training_args.do_train
				    and not training_args.overwrite_output_dir
				):
								raise ValueError(
								    F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
								    ''' --overwrite_output_dir to overcome.'''    )

				# Setup logging
				logging.basicConfig(
				    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''		,       datefmt='''%m/%d/%Y %H:%M:%S'''		,       level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN		,       )
				logger.warning(
				    '''Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'''		,       training_args.local_rank		,       training_args.device		,       training_args.n_gpu		,       bool(training_args.local_rank != -1    )		,       training_args.fpaa		,       )
				# Set the verbosity to info of the Transformers logger (on main process only):
				if is_main_process(training_args.local_rank    ):
								transformers.utils.logging.set_verbosity_info()
								transformers.utils.logging.enable_default_handler()
								transformers.utils.logging.enable_explicit_format()
				logger.info('''Training/evaluation parameters %s'''		,       lowerCamelCase_    )

				# Prepare Question-Answering task
				# Load pretrained model and tokenizer
				#
				# Distributed training:
				# The .from_pretrained methods guarantee that only one local process can concurrently
				# download model & vocab.

				lowercase__       			=      AutoConfig.from_pretrained(
				    model_args.config_name if model_args.config_name else model_args.model_name_or_path		,       cache_dir=model_args.cache_dir		,       )
				lowercase__       			=      AutoTokenizer.from_pretrained(
				    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path		,       cache_dir=model_args.cache_dir		,       use_fast=lowerCamelCase_		,       )
				lowercase__       			=      AutoModelForQuestionAnswering.from_pretrained(
				    model_args.model_name_or_path		,       from_tf=bool('''.ckpt''' in model_args.model_name_or_path    )		,       config=lowerCamelCase_		,       cache_dir=model_args.cache_dir		,       )

				# Get datasets
				lowercase__       			=      hasattr(model.config		,       '''lang2id'''    )
				lowercase__       			=      (
				    SquadDataset(
				        lowerCamelCase_		,       tokenizer=lowerCamelCase_		,       is_language_sensitive=lowerCamelCase_		,       cache_dir=model_args.cache_dir    )
				    if training_args.do_train
				    else None
				)
				lowercase__       			=      (
				    SquadDataset(
				        lowerCamelCase_		,       tokenizer=lowerCamelCase_		,       mode='''dev'''		,       is_language_sensitive=lowerCamelCase_		,       cache_dir=model_args.cache_dir		,       )
				    if training_args.do_eval
				    else None
				)

				# Data collator
				lowercase__       			=      DataCollatorWithPadding(lowerCamelCase_		,       pad_to_multiple_of=8    ) if training_args.fpaa else None

				# Initialize our Trainer
				lowercase__       			=      Trainer(
				    model=lowerCamelCase_		,       args=lowerCamelCase_		,       train_dataset=lowerCamelCase_		,       eval_dataset=lowerCamelCase_		,       data_collator=lowerCamelCase_		,       )

				# Training
				if training_args.do_train:
								trainer.train(
								    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path    ) else None    )
								trainer.save_model()
								# For convenience, we also re-save the tokenizer to the same directory,
								# so that you can share your model easily on huggingface.co/models =)
								if trainer.is_world_master():
												tokenizer.save_pretrained(training_args.output_dir    )






def   a      (						lowerCamelCase_    ):






				'''simple docstring'''
				main()


if __name__ == "__main__":
							main()







import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil






@pytest.fixture
def   a      (						lowerCamelCase_    ):






				'''simple docstring'''
				lowercase__       			=      tmp_path / '''text.txt'''
				lowercase__       			=      textwrap.dedent(
				    '''\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        '''    )
				with open(lowerCamelCase_		,       '''w'''		,       encoding='''utf-8'''    ) as f:
								f.write(lowerCamelCase_    )
				return str(lowerCamelCase_    )






@pytest.fixture
def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				lowercase__       			=      tmp_path / '''text_with_image.txt'''
				with open(lowerCamelCase_		,       '''w'''		,       encoding='''utf-8'''    ) as f:
								f.write(lowerCamelCase_    )
				return str(lowerCamelCase_    )






@pytest.mark.parametrize('''keep_linebreaks'''		,       [True, False]    )
def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				with open(lowerCamelCase_		,       encoding='''utf-8'''    ) as f:
								lowercase__       			=      f.read().splitlines(keepends=lowerCamelCase_    )
				lowercase__       			=      Text(keep_linebreaks=lowerCamelCase_		,       encoding='''utf-8'''    )
				lowercase__       			=      text._generate_tables([[text_file]]    )
				lowercase__       			=      pa.concat_tables([table for _, table in generator]    ).to_pydict()['''text''']
				assert generated_content == expected_content






@require_pil
def   a      (						lowerCamelCase_    ):






				'''simple docstring'''
				with open(lowerCamelCase_		,       encoding='''utf-8'''    ) as f:
								lowercase__       			=      f.read().splitlines()[0]
				lowercase__       			=      Text(encoding='''utf-8'''		,       features=Features({'''image''': Image()}    )    )
				lowercase__       			=      text._generate_tables([[text_file_with_image]]    )
				lowercase__       			=      pa.concat_tables([table for _, table in generator]    )
				assert pa_table.schema.field('''image'''    ).type == Image()()
				lowercase__       			=      pa_table.to_pydict()['''image''']
				assert generated_content == [{"path": image_file, "bytes": None}]






@pytest.mark.parametrize('''sample_by'''		,       ['''line''', '''paragraph''', '''document''']    )
def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				with open(lowerCamelCase_		,       encoding='''utf-8'''    ) as f:
								lowercase__       			=      f.read()
				if sample_by == "line":
								lowercase__       			=      expected_content.splitlines()
				elif sample_by == "paragraph":
								lowercase__       			=      expected_content.split('''\n\n'''    )
				elif sample_by == "document":
								lowercase__       			=      [expected_content]
				lowercase__       			=      Text(sample_by=lowerCamelCase_		,       encoding='''utf-8'''		,       chunksize=100    )
				lowercase__       			=      text._generate_tables([[text_file]]    )
				lowercase__       			=      pa.concat_tables([table for _, table in generator]    ).to_pydict()['''text''']
				assert generated_content == expected_content




