

"""simple docstring"""




import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


__UpperCAmelCase	=			logging.getLogger(__name__)







@dataclass
class        __lowercase     :
  snake_case_			    =    field(
      metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}   )
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}   )
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}   )
  snake_case_			    =    field(default=__lowerCamelCase		,     metadata={"""help""": """Set this flag to use fast tokenization."""}   )
  # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
  # or just modify its tokenizer_config.json.
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}		,     )







def 					lowerCAmelCase  (   ):
 '''simple docstring'''





 UpperCAmelCase__   :					List[str]   				=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

 if len(sys.argv ) == 2 and sys.argv[1].endswith(""".json""" ):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
  UpperCAmelCase__				, UpperCAmelCase__				, UpperCAmelCase__   :					List[str]   				=   parser.parse_json_file(json_file=os.path.abspath(sys.argv[1] ) )
 else:
  UpperCAmelCase__				, UpperCAmelCase__				, UpperCAmelCase__   :					List[Any]   				=   parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir )
     and os.listdir(training_args.output_dir )
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      F"Output directory ({training_args.output_dir}) already exists and is not empty. Use"
      """ --overwrite_output_dir to overcome.""" )

 # Setup logging
 logging.basicConfig(
     format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""							,      datefmt="""%m/%d/%Y %H:%M:%S"""							,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN							,      )
 logger.warning(
     """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""							,      training_args.local_rank							,      training_args.device							,      training_args.n_gpu							,      bool(training_args.local_rank != -1 )							,      training_args.fpaa							,      )
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank ):
  transformers.utils.logging.set_verbosity_info()
  transformers.utils.logging.enable_default_handler()
  transformers.utils.logging.enable_explicit_format()
 logger.info("""Training/evaluation parameters %s"""							,      __UpperCamelCase )

 # Prepare Question-Answering task
 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 UpperCAmelCase__   :					Tuple   				=   AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path							,      cache_dir=model_args.cache_dir							,      )
 UpperCAmelCase__   :					Dict   				=   AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path							,      cache_dir=model_args.cache_dir							,      use_fast=__UpperCamelCase							,      )
 UpperCAmelCase__   :					Optional[Any]   				=   AutoModelForQuestionAnswering.from_pretrained(
     model_args.model_name_or_path							,      from_tf=bool(""".ckpt""" in model_args.model_name_or_path )							,      config=__UpperCamelCase							,      cache_dir=model_args.cache_dir							,      )

 # Get datasets
 UpperCAmelCase__   :					Dict   				=   hasattr(model.config							,      """lang2id""" )
 UpperCAmelCase__   :					List[Any]   				=   (
     SquadDataset(
         __UpperCamelCase							,      tokenizer=__UpperCamelCase							,      is_language_sensitive=__UpperCamelCase							,      cache_dir=model_args.cache_dir )
     if training_args.do_train
     else None
 )
 UpperCAmelCase__   :					Dict   				=   (
     SquadDataset(
         __UpperCamelCase							,      tokenizer=__UpperCamelCase							,      mode="""dev"""							,      is_language_sensitive=__UpperCamelCase							,      cache_dir=model_args.cache_dir							,      )
     if training_args.do_eval
     else None
 )

 # Data collator
 UpperCAmelCase__   :					Union[str, Any]   				=   DataCollatorWithPadding(__UpperCamelCase							,      pad_to_multiple_of=8 ) if training_args.fpaa else None

 # Initialize our Trainer
 UpperCAmelCase__   :					Tuple   				=   Trainer(
     model=__UpperCamelCase							,      args=__UpperCamelCase							,      train_dataset=__UpperCamelCase							,      eval_dataset=__UpperCamelCase							,      data_collator=__UpperCamelCase							,      )

 # Training
 if training_args.do_train:
  trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path ) else None )
  trainer.save_model()
  # For convenience, we also re-save the tokenizer to the same directory,
  # so that you can share your model easily on huggingface.co/models =)
  if trainer.is_world_master():
   tokenizer.save_pretrained(training_args.output_dir )
def 					lowerCAmelCase  (   __UpperCamelCase ):
 '''simple docstring'''





 main()


if __name__ == "__main__":
      main()








"""simple docstring"""




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
      import PIL


__UpperCAmelCase	=			logging.get_logger(__name__)







class        __lowercase     (      __lowerCamelCase   ):
  snake_case_			    =    ["""pixel_values"""]

  def __init__(					self						:     Tuple   ,A						:     bool = True   ,A						:     Dict[str, int] = None   ,A						:     PILImageResampling = PIL.Image.NEAREST   ,A						:     bool = False   ,A						:     Dict[str, int] = None   ,A						:     Union[int, float] = 1 / 255   ,A						:     bool = False   ,A						:     bool = True   ,A						:     bool = True   ,A						:     Optional[Union[float, List[float]]] = None   ,A						:     Optional[Union[float, List[float]]] = None   ,A						:     bool = True   ,**A						:     List[str]   ,):






   '''simple docstring'''
   super().__init__(**A      )
   UpperCAmelCase__   :					Optional[Any]   				=   size if size is not None else {"""height""": 346, """width""": 346}
   UpperCAmelCase__   :					Union[str, Any]   				=   get_size_dict(A      )
   UpperCAmelCase__   :					Union[str, Any]   				=   crop_size if crop_size is not None else {"""height""": 289, """width""": 289}
   UpperCAmelCase__   :					Optional[Any]   				=   get_size_dict(A   ,param_name="""crop_size"""      )

   UpperCAmelCase__   :					Optional[int]   				=   do_resize
   UpperCAmelCase__   :					Optional[int]   				=   size
   UpperCAmelCase__   :					Optional[int]   				=   resample
   UpperCAmelCase__   :					List[str]   				=   do_center_crop
   UpperCAmelCase__   :					int   				=   crop_size
   UpperCAmelCase__   :					List[Any]   				=   do_rescale
   UpperCAmelCase__   :					List[str]   				=   rescale_factor
   UpperCAmelCase__   :					List[str]   				=   rescale_offset
   UpperCAmelCase__   :					Any   				=   do_normalize
   UpperCAmelCase__   :					Tuple   				=   image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
   UpperCAmelCase__   :					int   				=   image_std if image_std is not None else IMAGENET_STANDARD_STD
   UpperCAmelCase__   :					Optional[int]   				=   include_top

  def 				__lowercase    (					self						:     Tuple   ,A						:     np.ndarray   ,A						:     Dict[str, int]   ,A						:     PILImageResampling = PIL.Image.NEAREST   ,A						:     Optional[Union[str, ChannelDimension]] = None   ,**A						:     Optional[int]   ,):






   '''simple docstring'''
   UpperCAmelCase__   :					List[Any]   				=   get_size_dict(A      )
   if "height" not in size or "width" not in size:
    raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"      )
   return resize(
       A   ,size=(size["""height"""], size["""width"""])   ,resample=A   ,data_format=A   ,**A      )

  def 				__lowercase    (					self						:     Union[str, Any]   ,A						:     np.ndarray   ,A						:     Dict[str, int]   ,A						:     Optional[Union[str, ChannelDimension]] = None   ,**A						:     Union[str, Any]   ,):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[Any]   				=   get_size_dict(A      )
   if "height" not in size or "width" not in size:
    raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"      )
   return center_crop(A   ,size=(size["""height"""], size["""width"""])   ,data_format=A   ,**A      )

  def 				__lowercase    (					self						:     List[Any]   ,A						:     np.ndarray   ,A						:     Union[int, float]   ,A						:     bool = True   ,A						:     Optional[Union[str, ChannelDimension]] = None   ,**A						:     Any   ,):






   '''simple docstring'''
   if offset:
    UpperCAmelCase__   :					Tuple   				=   (image - 1_2_7.5) * scale
    if data_format is not None:
     UpperCAmelCase__   :					Any   				=   to_channel_dimension_format(A   ,A      )
    UpperCAmelCase__   :					Union[str, Any]   				=   rescaled_image.astype(np.floataa      )
   else:
    UpperCAmelCase__   :					List[str]   				=   rescale(A   ,scale=A   ,data_format=A   ,**A      )
   return rescaled_image

  def 				__lowercase    (					self						:     Optional[Any]   ,A						:     np.ndarray   ,A						:     Union[float, List[float]]   ,A						:     Union[float, List[float]]   ,A						:     Optional[Union[str, ChannelDimension]] = None   ,**A						:     List[Any]   ,):






   '''simple docstring'''
   return normalize(A   ,mean=A   ,std=A   ,data_format=A   ,**A      )




  def 				__lowercase    (					self						:     Optional[int]   ,A						:     ImageInput   ,A						:     bool = None   ,A						:     Dict[str, int] = None   ,A						:     str=None   ,A						:     bool = None   ,A						:     Dict[str, int] = None   ,A						:     bool = None   ,A						:     float = None   ,A						:     bool = None   ,A						:     bool = None   ,A						:     Optional[Union[float, List[float]]] = None   ,A						:     Optional[Union[float, List[float]]] = None   ,A						:     bool = None   ,A						:     Optional[Union[str, TensorType]] = None   ,A						:     ChannelDimension = ChannelDimension.FIRST   ,**A						:     Optional[int]   ,):






   '''simple docstring'''
   UpperCAmelCase__   :					List[str]   				=   do_resize if do_resize is not None else self.do_resize
   UpperCAmelCase__   :					Optional[int]   				=   resample if resample is not None else self.resample
   UpperCAmelCase__   :					List[str]   				=   do_center_crop if do_center_crop is not None else self.do_center_crop
   UpperCAmelCase__   :					List[str]   				=   do_rescale if do_rescale is not None else self.do_rescale
   UpperCAmelCase__   :					Any   				=   rescale_factor if rescale_factor is not None else self.rescale_factor
   UpperCAmelCase__   :					str   				=   rescale_offset if rescale_offset is not None else self.rescale_offset
   UpperCAmelCase__   :					Union[str, Any]   				=   do_normalize if do_normalize is not None else self.do_normalize
   UpperCAmelCase__   :					int   				=   image_mean if image_mean is not None else self.image_mean
   UpperCAmelCase__   :					Any   				=   image_std if image_std is not None else self.image_std
   UpperCAmelCase__   :					List[Any]   				=   include_top if include_top is not None else self.include_top

   UpperCAmelCase__   :					List[Any]   				=   size if size is not None else self.size
   UpperCAmelCase__   :					int   				=   get_size_dict(A      )
   UpperCAmelCase__   :					Any   				=   crop_size if crop_size is not None else self.crop_size
   UpperCAmelCase__   :					str   				=   get_size_dict(A   ,param_name="""crop_size"""      )

   UpperCAmelCase__   :					Any   				=   make_list_of_images(A      )

   if not valid_images(A      ):
    raise ValueError(
        """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
        """torch.Tensor, tf.Tensor or jax.ndarray."""      )

   if do_resize and size is None or resample is None:
    raise ValueError("""Size and resample must be specified if do_resize is True."""      )

   if do_center_crop and crop_size is None:
    raise ValueError("""Crop size must be specified if do_center_crop is True."""      )

   if do_rescale and rescale_factor is None:
    raise ValueError("""Rescale factor must be specified if do_rescale is True."""      )

   if do_normalize and (image_mean is None or image_std is None):
    raise ValueError("""Image mean and std must be specified if do_normalize is True."""      )

   # All transformations expect numpy arrays.
   UpperCAmelCase__   :					List[str]   				=   [to_numpy_array(A      ) for image in images]

   if do_resize:
    UpperCAmelCase__   :					str   				=   [self.resize(image=A   ,size=A   ,resample=A      ) for image in images]

   if do_center_crop:
    UpperCAmelCase__   :					Tuple   				=   [self.center_crop(image=A   ,size=A      ) for image in images]

   if do_rescale:
    UpperCAmelCase__   :					int   				=   [self.rescale(image=A   ,scale=A   ,offset=A      ) for image in images]

   if do_normalize:
    UpperCAmelCase__   :					int   				=   [self.normalize(image=A   ,mean=A   ,std=A      ) for image in images]

   if include_top:
    UpperCAmelCase__   :					Dict   				=   [self.normalize(image=A   ,mean=[0, 0, 0]   ,std=A      ) for image in images]

   UpperCAmelCase__   :					Optional[Any]   				=   [to_channel_dimension_format(A   ,A      ) for image in images]

   UpperCAmelCase__   :					Dict   				=   {"""pixel_values""": images}
   return BatchFeature(data=A   ,tensor_type=A      )





