






import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


lowercase  							= logging.getLogger(__name__)




@dataclass
class 			UpperCamelCase_				:



	'''simple docstring'''




	lowerCAmelCase													=			field(
	    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}    )
	lowerCAmelCase													=			field(
	    default=UpperCAmelCase_	, metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}    )
	lowerCAmelCase													=			field(
	    default=UpperCAmelCase_	, metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}    )
	lowerCAmelCase													=			field(default=UpperCAmelCase_	, metadata={'''help''': '''Set this flag to use fast tokenization.'''}    )
	# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
	# or just modify its tokenizer_config.json.
	lowerCAmelCase													=			field(
	    default=UpperCAmelCase_	, metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}	, )






def 						__UpperCAmelCase      (					):
							snake_case_           =							HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))

							if len(sys.argv) == 2 and sys.argv[1].endswith('.json'):
														# If we pass only one argument to the script and it's the path to a json file,
														# let's parse it to get our arguments.
														snake_case_      ,		snake_case_      ,		snake_case_           =							parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
							else:
														snake_case_      ,		snake_case_      ,		snake_case_           =							parser.parse_args_into_dataclasses()

							if (
							    os.path.exists(training_args.output_dir)
							    and os.listdir(training_args.output_dir)
							    and training_args.do_train
							    and not training_args.overwrite_output_dir
							):
														raise ValueError(
														    f'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
														    ' --overwrite_output_dir to overcome.')

							# Setup logging
							logging.basicConfig(
							    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'							,					datefmt='%m/%d/%Y %H:%M:%S'							,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN							,					)
							logger.warning(
							    'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'							,					training_args.local_rank							,					training_args.device							,					training_args.n_gpu							,					bool(training_args.local_rank != -1)							,					training_args.fpaa							,					)
							# Set the verbosity to info of the Transformers logger (on main process only):
							if is_main_process(training_args.local_rank):
														transformers.utils.logging.set_verbosity_info()
														transformers.utils.logging.enable_default_handler()
														transformers.utils.logging.enable_explicit_format()
							logger.info('Training/evaluation parameters %s'							,					lowercase_)

							# Prepare Question-Answering task
							# Load pretrained model and tokenizer
							#
							# Distributed training:
							# The .from_pretrained methods guarantee that only one local process can concurrently
							# download model & vocab.

							snake_case_           =							AutoConfig.from_pretrained(
							    model_args.config_name if model_args.config_name else model_args.model_name_or_path							,					cache_dir=model_args.cache_dir							,					)
							snake_case_           =							AutoTokenizer.from_pretrained(
							    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path							,					cache_dir=model_args.cache_dir							,					use_fast=lowercase_							,					)
							snake_case_           =							AutoModelForQuestionAnswering.from_pretrained(
							    model_args.model_name_or_path							,					from_tf=bool('.ckpt' in model_args.model_name_or_path)							,					config=lowercase_							,					cache_dir=model_args.cache_dir							,					)

							# Get datasets
							snake_case_           =							hasattr(model.config							,					'lang2id')
							snake_case_           =							(
							    SquadDataset(
							        lowercase_							,					tokenizer=lowercase_							,					is_language_sensitive=lowercase_							,					cache_dir=model_args.cache_dir)
							    if training_args.do_train
							    else None
							)
							snake_case_           =							(
							    SquadDataset(
							        lowercase_							,					tokenizer=lowercase_							,					mode='dev'							,					is_language_sensitive=lowercase_							,					cache_dir=model_args.cache_dir							,					)
							    if training_args.do_eval
							    else None
							)

							# Data collator
							snake_case_           =							DataCollatorWithPadding(lowercase_							,					pad_to_multiple_of=8) if training_args.fpaa else None

							# Initialize our Trainer
							snake_case_           =							Trainer(
							    model=lowercase_							,					args=lowercase_							,					train_dataset=lowercase_							,					eval_dataset=lowercase_							,					data_collator=lowercase_							,					)

							# Training
							if training_args.do_train:
														trainer.train(
														    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None)
														trainer.save_model()
														# For convenience, we also re-save the tokenizer to the same directory,
														# so that you can share your model easily on huggingface.co/models =)
														if trainer.is_world_master():
																					tokenizer.save_pretrained(training_args.output_dir)





def 						__UpperCAmelCase      (					a_):
							main()


if __name__ == "__main__":
	main()













import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
	import transformer_engine.pytorch as te





def 						__UpperCAmelCase      (					a_							,					a_=True							,					a_=True							,					a_=True):
							if not is_fpa_available():
														raise ImportError('Using `convert_model` requires transformer_engine to be installed.')
							for name, module in model.named_children():
														if isinstance(a_							,					nn.Linear) and to_transformer_engine and _convert_linear:
																					# Return early if the linear layer weights are not multiples of 16
																					if any(p % 16 != 0 for p in module.weight.shape):
																												return
																					snake_case_           =							module.bias is not None
																					snake_case_           =							te.Linear(
																					    module.in_features							,					module.out_features							,					bias=a_							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					if has_bias:
																												snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														elif isinstance(a_							,					nn.LayerNorm) and to_transformer_engine and _convert_ln:
																					snake_case_           =							te.LayerNorm(module.normalized_shape[0]							,					eps=module.eps							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														elif isinstance(a_							,					te.Linear) and not to_transformer_engine and _convert_linear:
																					snake_case_           =							module.bias is not None
																					snake_case_           =							nn.Linear(
																					    module.in_features							,					module.out_features							,					bias=a_							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					if has_bias:
																												snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														elif isinstance(a_							,					te.LayerNorm) and not to_transformer_engine and _convert_ln:
																					snake_case_           =							nn.LayerNorm(module.normalized_shape[0]							,					eps=module.eps							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														else:
																					convert_model(
																					    a_							,					to_transformer_engine=a_							,					_convert_linear=_convert_linear							,					_convert_ln=_convert_ln							,					)





def 						__UpperCAmelCase      (					a_):
							if not is_fpa_available():
														raise ImportError('Using `has_transformer_engine_layers` requires transformer_engine to be installed.')
							for m in model.modules():
														if isinstance(a_							,					(te.LayerNorm, te.Linear)):
																					return True
							return False





