import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


lowerCAmelCase__					:       Tuple		=       logging.getLogger(__name__)





@dataclass
class 			__snake_case  :
					__lowerCamelCase	 =       field(
					    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}     )
					__lowerCamelCase	 =       field(
					    default=_lowerCamelCase   ,metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}     )
					__lowerCamelCase	 =       field(
					    default=_lowerCamelCase   ,metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}     )
					__lowerCamelCase	 =       field(default=_lowerCamelCase   ,metadata={"""help""": """Set this flag to use fast tokenization."""}     )
					# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
					# or just modify its tokenizer_config.json.
					__lowerCamelCase	 =       field(
					    default=_lowerCamelCase   ,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}   ,)




def 			UpperCamelCase__						(    )       -> Optional[Any]:
				# See all possible arguments in src/transformers/training_args.py
				# or by passing the --help flag to this script.
				# We now keep distinct sets of args, for a cleaner separation of concerns.

				snake_case__ :     Optional[Any] 							=				HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)   )

				if len(sys.argv   ) == 2 and sys.argv[1].endswith('.json'   ):
								# If we pass only one argument to the script and it's the path to a json file,
								# let's parse it to get our arguments.
								snake_case__							,  snake_case__							,  snake_case__ :     Tuple 							=				parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]   )   )
				else:
								snake_case__							,  snake_case__							,  snake_case__ :     Optional[int] 							=				parser.parse_args_into_dataclasses()

				if (
				    os.path.exists(training_args.output_dir   )
				    and os.listdir(training_args.output_dir   )
				    and training_args.do_train
				    and not training_args.overwrite_output_dir
				):
								raise ValueError(
								    F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
								    ' --overwrite_output_dir to overcome.'   )

				# Setup logging
				logging.basicConfig(
				    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'      ,			datefmt='%m/%d/%Y %H:%M:%S'      ,			level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN      ,			)
				logger.warning(
				    'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'      ,			training_args.local_rank      ,			training_args.device      ,			training_args.n_gpu      ,			bool(training_args.local_rank != -1   )      ,			training_args.fpaa      ,			)
				# Set the verbosity to info of the Transformers logger (on main process only):
				if is_main_process(training_args.local_rank   ):
								transformers.utils.logging.set_verbosity_info()
								transformers.utils.logging.enable_default_handler()
								transformers.utils.logging.enable_explicit_format()
				logger.info('Training/evaluation parameters %s'      ,			A__   )

				# Prepare Question-Answering task
				# Load pretrained model and tokenizer
				#
				# Distributed training:
				# The .from_pretrained methods guarantee that only one local process can concurrently
				# download model & vocab.

				snake_case__ :     Optional[int] 							=				AutoConfig.from_pretrained(
				    model_args.config_name if model_args.config_name else model_args.model_name_or_path      ,			cache_dir=model_args.cache_dir      ,			)
				snake_case__ :     str 							=				AutoTokenizer.from_pretrained(
				    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path      ,			cache_dir=model_args.cache_dir      ,			use_fast=A__      ,			)
				snake_case__ :     Optional[int] 							=				AutoModelForQuestionAnswering.from_pretrained(
				    model_args.model_name_or_path      ,			from_tf=bool('.ckpt' in model_args.model_name_or_path   )      ,			config=A__      ,			cache_dir=model_args.cache_dir      ,			)

				# Get datasets
				snake_case__ :     Dict 							=				hasattr(model.config      ,			'lang2id'   )
				snake_case__ :     List[str] 							=				(
				    SquadDataset(
				        A__      ,			tokenizer=A__      ,			is_language_sensitive=A__      ,			cache_dir=model_args.cache_dir   )
				    if training_args.do_train
				    else None
				)
				snake_case__ :     str 							=				(
				    SquadDataset(
				        A__      ,			tokenizer=A__      ,			mode='dev'      ,			is_language_sensitive=A__      ,			cache_dir=model_args.cache_dir      ,			)
				    if training_args.do_eval
				    else None
				)

				# Data collator
				snake_case__ :     Dict 							=				DataCollatorWithPadding(A__      ,			pad_to_multiple_of=8   ) if training_args.fpaa else None

				# Initialize our Trainer
				snake_case__ :     str 							=				Trainer(
				    model=A__      ,			args=A__      ,			train_dataset=A__      ,			eval_dataset=A__      ,			data_collator=A__      ,			)

				# Training
				if training_args.do_train:
								trainer.train(
								    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path   ) else None   )
								trainer.save_model()
								# For convenience, we also re-save the tokenizer to the same directory,
								# so that you can share your model easily on huggingface.co/models =)
								if trainer.is_world_master():
												tokenizer.save_pretrained(training_args.output_dir   )
def 			UpperCamelCase__						(    A__   )       -> Tuple:
				# For xla_spawn (TPUs)
				main()


if __name__ == "__main__":
				main()



from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers.models.openai.modeling_tf_openai import (
				    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFOpenAIGPTDoubleHeadsModel,
				    TFOpenAIGPTForSequenceClassification,
				    TFOpenAIGPTLMHeadModel,
				    TFOpenAIGPTModel,
				)





class 			__snake_case  :




					def __init__(       self   ,	__UpperCamelCase   ,	)					->  Tuple:



									'''simple docstring'''







									snake_case__ :     List[str] 							=				parent
									snake_case__ :     Optional[int] 							=				13
									snake_case__ :     Tuple 							=				7
									snake_case__ :     Optional[int] 							=				True
									snake_case__ :     int 							=				True
									snake_case__ :     int 							=				True
									snake_case__ :     int 							=				True
									snake_case__ :     Union[str, Any] 							=				True
									snake_case__ :     Optional[Any] 							=				99
									snake_case__ :     Dict 							=				32
									snake_case__ :     Union[str, Any] 							=				2
									snake_case__ :     Tuple 							=				4
									snake_case__ :     List[str] 							=				37
									snake_case__ :     Optional[int] 							=				'gelu'
									snake_case__ :     List[str] 							=				0.1
									snake_case__ :     Union[str, Any] 							=				0.1
									snake_case__ :     str 							=				512
									snake_case__ :     str 							=				16
									snake_case__ :     Dict 							=				2
									snake_case__ :     List[Any] 							=				0.0_2
									snake_case__ :     Union[str, Any] 							=				3
									snake_case__ :     Optional[Any] 							=				4
									snake_case__ :     List[Any] 							=				None
									snake_case__ :     Union[str, Any] 							=				self.vocab_size - 1




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     Any 							=				ids_tensor([self.batch_size, self.seq_length]   ,	self.vocab_size		)

									snake_case__ :     List[Any] 							=				None
									if self.use_input_mask:
													snake_case__ :     str 							=				random_attention_mask([self.batch_size, self.seq_length]		)

									snake_case__ :     List[str] 							=				None
									if self.use_token_type_ids:
													snake_case__ :     List[Any] 							=				ids_tensor([self.batch_size, self.seq_length]   ,	self.type_vocab_size		)

									snake_case__ :     Dict 							=				None
									if self.use_mc_token_ids:
													snake_case__ :     Any 							=				ids_tensor([self.batch_size, self.num_choices]   ,	self.seq_length		)

									snake_case__ :     List[Any] 							=				None
									snake_case__ :     Dict 							=				None
									snake_case__ :     str 							=				None
									if self.use_labels:
													snake_case__ :     Tuple 							=				ids_tensor([self.batch_size]   ,	self.type_sequence_label_size		)
													snake_case__ :     Tuple 							=				ids_tensor([self.batch_size, self.seq_length]   ,	self.num_labels		)
													snake_case__ :     Tuple 							=				ids_tensor([self.batch_size]   ,	self.num_choices		)

									snake_case__ :     str 							=				OpenAIGPTConfig(
									    vocab_size=self.vocab_size   ,	n_embd=self.hidden_size   ,	n_layer=self.num_hidden_layers   ,	n_head=self.num_attention_heads   ,	n_positions=self.max_position_embeddings   ,	pad_token_id=self.pad_token_id   ,	)

									snake_case__ :     Union[str, Any] 							=				ids_tensor([self.num_hidden_layers, self.num_attention_heads]   ,	2		)

									return (
									    config,
									    input_ids,
									    input_mask,
									    head_mask,
									    token_type_ids,
									    mc_token_ids,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  List[str]:



									'''simple docstring'''







									snake_case__ :     List[Any] 							=				TFOpenAIGPTModel(config=__UpperCamelCase		)
									snake_case__ :     str 							=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
									snake_case__ :     Dict 							=				model(__UpperCamelCase		)

									snake_case__ :     Tuple 							=				[input_ids, input_mask]
									snake_case__ :     List[str] 							=				model(__UpperCamelCase		)

									snake_case__ :     Tuple 							=				model(__UpperCamelCase		)

									self.parent.assertEqual(result.last_hidden_state.shape   ,	(self.batch_size, self.seq_length, self.hidden_size)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  Optional[int]:



									'''simple docstring'''







									snake_case__ :     List[str] 							=				TFOpenAIGPTLMHeadModel(config=__UpperCamelCase		)
									snake_case__ :     Optional[int] 							=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
									snake_case__ :     List[str] 							=				model(__UpperCamelCase		)
									self.parent.assertEqual(result.logits.shape   ,	(self.batch_size, self.seq_length, self.vocab_size)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  Tuple:



									'''simple docstring'''







									snake_case__ :     Dict 							=				TFOpenAIGPTDoubleHeadsModel(config=__UpperCamelCase		)

									snake_case__ :     Optional[int] 							=				tf.tile(tf.expand_dims(__UpperCamelCase   ,	1		)   ,	(1, self.num_choices, 1)		)
									snake_case__ :     Tuple 							=				tf.tile(tf.expand_dims(__UpperCamelCase   ,	1		)   ,	(1, self.num_choices, 1)		)
									snake_case__ :     Any 							=				tf.tile(tf.expand_dims(__UpperCamelCase   ,	1		)   ,	(1, self.num_choices, 1)		)

									snake_case__ :     Dict 							=				{
									    'input_ids': multiple_choice_inputs_ids,
									    'mc_token_ids': mc_token_ids,
									    'attention_mask': multiple_choice_input_mask,
									    'token_type_ids': multiple_choice_token_type_ids,
									}
									snake_case__ :     Tuple 							=				model(__UpperCamelCase		)
									self.parent.assertEqual(
									    result.logits.shape   ,	(self.batch_size, self.num_choices, self.seq_length, self.vocab_size)		)
									self.parent.assertEqual(result.mc_logits.shape   ,	(self.batch_size, self.num_choices)		)




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	*__UpperCamelCase		)					->  Tuple:



									'''simple docstring'''







									snake_case__ :     int 							=				self.num_labels
									snake_case__ :     int 							=				ids_tensor([self.batch_size]   ,	self.type_sequence_label_size		)
									snake_case__ :     str 							=				{
									    'input_ids': input_ids,
									    'attention_mask': input_mask,
									    'token_type_ids': token_type_ids,
									    'labels': sequence_labels,
									}
									snake_case__ :     str 							=				TFOpenAIGPTForSequenceClassification(__UpperCamelCase		)
									snake_case__ :     int 							=				model(__UpperCamelCase		)
									self.parent.assertEqual(result.logits.shape   ,	(self.batch_size, self.num_labels)		)




					def       __a      (       self		)					->  List[str]:



									'''simple docstring'''







									snake_case__ :     Dict 							=				self.prepare_config_and_inputs()

									(
									    (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  (
									    snake_case__
									)							,  
									) :     Any 							=				config_and_inputs

									snake_case__ :     List[str] 							=				{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict





@require_tf
class 			__snake_case  (      _lowerCamelCase   ,_lowerCamelCase   ,unittest.TestCase     ):
					__lowerCamelCase	 =       (
					    (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
					    if is_tf_available()
					    else ()
					)
					__lowerCamelCase	 =       (
					    (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
					)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
					__lowerCamelCase	 =       (
					    {
					        """feature-extraction""": TFOpenAIGPTModel,
					        """text-classification""": TFOpenAIGPTForSequenceClassification,
					        """text-generation""": TFOpenAIGPTLMHeadModel,
					        """zero-shot""": TFOpenAIGPTForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					__lowerCamelCase	 =       False
					__lowerCamelCase	 =       False




					def       __a      (       self   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase   ,	__UpperCamelCase		)					->  int:



									'''simple docstring'''







									if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
													# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
													# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
													# tiny config could not be created.
													return True

									return False




					def       __a      (       self		)					->  Any:



									'''simple docstring'''







									snake_case__ :     Tuple 							=				TFOpenAIGPTModelTester(self		)
									snake_case__ :     Any 							=				ConfigTester(self   ,	config_class=__UpperCamelCase   ,	n_embd=37		)




					def       __a      (       self		)					->  int:



									'''simple docstring'''







									self.config_tester.run_common_tests()




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     Optional[Any] 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_model(*__UpperCamelCase		)




					def       __a      (       self		)					->  int:



									'''simple docstring'''







									snake_case__ :     str 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_lm_head(*__UpperCamelCase		)




					def       __a      (       self		)					->  List[Any]:



									'''simple docstring'''







									snake_case__ :     Dict 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_double_head(*__UpperCamelCase		)




					def       __a      (       self		)					->  Dict:



									'''simple docstring'''







									snake_case__ :     str 							=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__UpperCamelCase		)




					@slow
					def       __a      (       self		)					->  Union[str, Any]:



									'''simple docstring'''







									for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													snake_case__ :     Optional[Any] 							=				TFOpenAIGPTModel.from_pretrained(__UpperCamelCase		)
													self.assertIsNotNone(__UpperCamelCase		)





@require_tf
class 			__snake_case  (      unittest.TestCase     ):




					@slow
					def       __a      (       self		)					->  Union[str, Any]:



									'''simple docstring'''







									snake_case__ :     Dict 							=				TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt'		)
									snake_case__ :     Any 							=				tf.convert_to_tensor([[481, 4735, 544]]   ,	dtype=tf.intaa		)  # the president is
									snake_case__ :     int 							=				[
									    481,
									    4735,
									    544,
									    246,
									    963,
									    870,
									    762,
									    239,
									    244,
									    40477,
									    244,
									    249,
									    719,
									    881,
									    487,
									    544,
									    240,
									    244,
									    603,
									    481,
									]  # the president is a very good man. " \n " i\'m sure he is, " said the

									snake_case__ :     int 							=				model.generate(__UpperCamelCase   ,	do_sample=__UpperCamelCase		)
									self.assertListEqual(output_ids[0].numpy().tolist()   ,	__UpperCamelCase		)


