

"""simple docstring"""




import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


__UpperCAmelCase	=			logging.get_logger(__name__)

__UpperCAmelCase	=			OrderedDict(
    [
        ('align', 'AlignProcessor'),
        ('altclip', 'AltCLIPProcessor'),
        ('bark', 'BarkProcessor'),
        ('blip', 'BlipProcessor'),
        ('blip-2', 'Blip2Processor'),
        ('bridgetower', 'BridgeTowerProcessor'),
        ('chinese_clip', 'ChineseCLIPProcessor'),
        ('clap', 'ClapProcessor'),
        ('clip', 'CLIPProcessor'),
        ('clipseg', 'CLIPSegProcessor'),
        ('flava', 'FlavaProcessor'),
        ('git', 'GitProcessor'),
        ('groupvit', 'CLIPProcessor'),
        ('hubert', 'Wav2Vec2Processor'),
        ('instructblip', 'InstructBlipProcessor'),
        ('layoutlmv2', 'LayoutLMv2Processor'),
        ('layoutlmv3', 'LayoutLMv3Processor'),
        ('markuplm', 'MarkupLMProcessor'),
        ('mctct', 'MCTCTProcessor'),
        ('mgp-str', 'MgpstrProcessor'),
        ('oneformer', 'OneFormerProcessor'),
        ('owlvit', 'OwlViTProcessor'),
        ('pix2struct', 'Pix2StructProcessor'),
        ('sam', 'SamProcessor'),
        ('sew', 'Wav2Vec2Processor'),
        ('sew-d', 'Wav2Vec2Processor'),
        ('speech_to_text', 'Speech2TextProcessor'),
        ('speech_to_text_2', 'Speech2Text2Processor'),
        ('speecht5', 'SpeechT5Processor'),
        ('trocr', 'TrOCRProcessor'),
        ('tvlt', 'TvltProcessor'),
        ('unispeech', 'Wav2Vec2Processor'),
        ('unispeech-sat', 'Wav2Vec2Processor'),
        ('vilt', 'ViltProcessor'),
        ('vision-text-dual-encoder', 'VisionTextDualEncoderProcessor'),
        ('wav2vec2', 'Wav2Vec2Processor'),
        ('wav2vec2-conformer', 'Wav2Vec2Processor'),
        ('wavlm', 'Wav2Vec2Processor'),
        ('whisper', 'WhisperProcessor'),
        ('xclip', 'XCLIPProcessor'),
    ]
)

__UpperCAmelCase	=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)
def 					lowerCAmelCase  (   __UpperCamelCase ):
 '''simple docstring'''





 for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
  if class_name in processors:
   UpperCAmelCase__   :					Dict   				=   model_type_to_module_name(__UpperCamelCase )

   UpperCAmelCase__   :					Dict   				=   importlib.import_module(F".{module_name}"							,      """transformers.models""" )
   try:
    return getattr(__UpperCamelCase							,      __UpperCamelCase )
   except AttributeError:
    continue

 for processor in PROCESSOR_MAPPING._extra_content.values():
  if getattr(__UpperCamelCase							,      """__name__"""							,      __UpperCamelCase ) == class_name:
   return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
 UpperCAmelCase__   :					List[str]   				=   importlib.import_module("""transformers""" )
 if hasattr(__UpperCamelCase							,      __UpperCamelCase ):
  return getattr(__UpperCamelCase							,      __UpperCamelCase )

 return None







class        __lowercase     :

  def __init__(					self						:     Optional[Any]      ):






   '''simple docstring'''
   raise EnvironmentError(
       """AutoProcessor is designed to be instantiated """
       """using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method."""      )

  @classmethod
  @replace_list_option_in_docstrings(A      )
  def 				__lowercase    (					cls						:     Dict   ,A						:     int   ,**A						:     Optional[int]      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[Any]   				=   kwargs.pop("""config"""   ,A      )
   UpperCAmelCase__   :					Union[str, Any]   				=   kwargs.pop("""trust_remote_code"""   ,A      )
   UpperCAmelCase__   :					Union[str, Any]   				=   True

   UpperCAmelCase__   :					Any   				=   None
   UpperCAmelCase__   :					Union[str, Any]   				=   None

   # First, let's see if we have a preprocessor config.
   # Filter the kwargs for `get_file_from_repo`.
   UpperCAmelCase__   :					int   				=   {
       key: kwargs[key] for key in inspect.signature(A      ).parameters.keys() if key in kwargs
   }
   # Let's start by checking whether the processor class is saved in an image processor
   UpperCAmelCase__   :					Optional[Any]   				=   get_file_from_repo(
       A   ,A   ,**A      )
   if preprocessor_config_file is not None:
    UpperCAmelCase__   :					str   				=   ImageProcessingMixin.get_image_processor_dict(A   ,**A      )
    UpperCAmelCase__   :					Any   				=   config_dict.get("""processor_class"""   ,A      )
    if "AutoProcessor" in config_dict.get("""auto_map"""   ,{}      ):
     UpperCAmelCase__   :					Optional[Any]   				=   config_dict["""auto_map"""]["""AutoProcessor"""]

        # If not found, let's check whether the processor class is saved in a feature extractor config
   if preprocessor_config_file is not None and processor_class is None:
    UpperCAmelCase__   :					Optional[int]   				=   FeatureExtractionMixin.get_feature_extractor_dict(A   ,**A      )
    UpperCAmelCase__   :					List[Any]   				=   config_dict.get("""processor_class"""   ,A      )
    if "AutoProcessor" in config_dict.get("""auto_map"""   ,{}      ):
     UpperCAmelCase__   :					str   				=   config_dict["""auto_map"""]["""AutoProcessor"""]

   if processor_class is None:
    # Next, let's check whether the processor class is saved in a tokenizer
    UpperCAmelCase__   :					Tuple   				=   get_file_from_repo(
        A   ,A   ,**A      )
    if tokenizer_config_file is not None:
     with open(A   ,encoding="""utf-8"""      ) as reader:
      UpperCAmelCase__   :					List[Any]   				=   json.load(A      )

     UpperCAmelCase__   :					List[Any]   				=   config_dict.get("""processor_class"""   ,A      )
     if "AutoProcessor" in config_dict.get("""auto_map"""   ,{}      ):
      UpperCAmelCase__   :					Optional[int]   				=   config_dict["""auto_map"""]["""AutoProcessor"""]

   if processor_class is None:
    # Otherwise, load config, if it can be loaded.
    if not isinstance(A   ,A      ):
     UpperCAmelCase__   :					Tuple   				=   AutoConfig.from_pretrained(
         A   ,trust_remote_code=A   ,**A      )

    # And check if the config contains the processor class.
    UpperCAmelCase__   :					Tuple   				=   getattr(A   ,"""processor_class"""   ,A      )
    if hasattr(A   ,"""auto_map"""      ) and "AutoProcessor" in config.auto_map:
     UpperCAmelCase__   :					List[str]   				=   config.auto_map["""AutoProcessor"""]

   if processor_class is not None:
    UpperCAmelCase__   :					List[Any]   				=   processor_class_from_name(A      )

   UpperCAmelCase__   :					Optional[int]   				=   processor_auto_map is not None
   UpperCAmelCase__   :					Union[str, Any]   				=   processor_class is not None or type(A      ) in PROCESSOR_MAPPING
   UpperCAmelCase__   :					Optional[Any]   				=   resolve_trust_remote_code(
       A   ,A   ,A   ,A      )

   if has_remote_code and trust_remote_code:
    UpperCAmelCase__   :					str   				=   get_class_from_dynamic_module(
        A   ,A   ,**A      )
    UpperCAmelCase__   :					str   				=   kwargs.pop("""code_revision"""   ,A      )
    if os.path.isdir(A      ):
     processor_class.register_for_auto_class()
    return processor_class.from_pretrained(
        A   ,trust_remote_code=A   ,**A      )
   elif processor_class is not None:
    return processor_class.from_pretrained(
        A   ,trust_remote_code=A   ,**A      )
   # Last try: we use the PROCESSOR_MAPPING.
   elif type(A      ) in PROCESSOR_MAPPING:
    return PROCESSOR_MAPPING[type(A      )].from_pretrained(A   ,**A      )

   # At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
   # tokenizer.
   try:
    return AutoTokenizer.from_pretrained(
        A   ,trust_remote_code=A   ,**A      )
   except Exception:
    try:
     return AutoImageProcessor.from_pretrained(
         A   ,trust_remote_code=A   ,**A      )
    except Exception:
     pass

    try:
     return AutoFeatureExtractor.from_pretrained(
         A   ,trust_remote_code=A   ,**A      )
    except Exception:
     pass

   raise ValueError(
       f"Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a "
       """tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains"""
       """the files of at least one of those processing classes."""      )




  @staticmethod
  def 				__lowercase    (					A						:     Optional[int]   ,A						:     List[str]      ):






   '''simple docstring'''
   PROCESSOR_MAPPING.register(A   ,A      )








"""simple docstring"""




import logging
import os
from dataclasses import dataclass, field
from typing import Dict, List, Optional

import numpy as np
import torch
from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import is_main_process


__UpperCAmelCase	=			logging.getLogger(__name__)







@dataclass
class        __lowercase     :
  snake_case_			    =    field(
      metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}   )
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}   )
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}   )
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}		,     )







@dataclass
class        __lowercase     :
  snake_case_			    =    field(
      metadata={"""help""": """The name of the task to train selected in the list: """ + """, """.join(hans_processors.keys()   )}   )
  snake_case_			    =    field(
      metadata={"""help""": """The input data dir. Should contain the .tsv files (or other data files) for the task."""}   )
  snake_case_			    =    field(
      default=1_2_8		,     metadata={
          """help""": (
              """The maximum total input sequence length after tokenization. Sequences longer """
              """than this will be truncated, sequences shorter will be padded."""
          )
      }		,     )
  snake_case_			    =    field(
      default=__lowerCamelCase		,     metadata={"""help""": """Overwrite the cached training and evaluation sets"""}   )







def 					lowerCAmelCase  (   __UpperCamelCase ):
 '''simple docstring'''





 UpperCAmelCase__   :					Optional[Any]   				=   default_data_collator(__UpperCamelCase )
 UpperCAmelCase__   :					List[str]   				=   batch.pop("""pairID"""							,      __UpperCamelCase )
 return batch
def 					lowerCAmelCase  (   ):
 '''simple docstring'''





 UpperCAmelCase__   :					Dict   				=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )
 UpperCAmelCase__				, UpperCAmelCase__				, UpperCAmelCase__   :					Union[str, Any]   				=   parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir )
     and os.listdir(training_args.output_dir )
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      F"Output directory ({training_args.output_dir}) already exists and is not empty. Use"
      """ --overwrite_output_dir to overcome.""" )

 # Setup logging
 logging.basicConfig(
     format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""							,      datefmt="""%m/%d/%Y %H:%M:%S"""							,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN							,      )
 logger.warning(
     """Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"""							,      training_args.local_rank							,      training_args.device							,      training_args.n_gpu							,      bool(training_args.local_rank != -1 )							,      training_args.fpaa							,      )
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank ):
  transformers.utils.logging.set_verbosity_info()
  transformers.utils.logging.enable_default_handler()
  transformers.utils.logging.enable_explicit_format()
 logger.info("""Training/evaluation parameters %s"""							,      __UpperCamelCase )

 # Set seed
 set_seed(training_args.seed )

 try:
  UpperCAmelCase__   :					Optional[int]   				=   hans_tasks_num_labels[data_args.task_name]
 except KeyError:
  raise ValueError("""Task not found: %s""" % (data_args.task_name) )

 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 UpperCAmelCase__   :					Tuple   				=   AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path							,      num_labels=__UpperCamelCase							,      finetuning_task=data_args.task_name							,      cache_dir=model_args.cache_dir							,      )
 UpperCAmelCase__   :					int   				=   AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path							,      cache_dir=model_args.cache_dir							,      )
 UpperCAmelCase__   :					List[Any]   				=   AutoModelForSequenceClassification.from_pretrained(
     model_args.model_name_or_path							,      from_tf=bool(""".ckpt""" in model_args.model_name_or_path )							,      config=__UpperCamelCase							,      cache_dir=model_args.cache_dir							,      )

 # Get datasets
 UpperCAmelCase__   :					Optional[int]   				=   (
     HansDataset(
         data_dir=data_args.data_dir							,      tokenizer=__UpperCamelCase							,      task=data_args.task_name							,      max_seq_length=data_args.max_seq_length							,      overwrite_cache=data_args.overwrite_cache							,      )
     if training_args.do_train
     else None
 )
 UpperCAmelCase__   :					str   				=   (
     HansDataset(
         data_dir=data_args.data_dir							,      tokenizer=__UpperCamelCase							,      task=data_args.task_name							,      max_seq_length=data_args.max_seq_length							,      overwrite_cache=data_args.overwrite_cache							,      evaluate=__UpperCamelCase							,      )
     if training_args.do_eval
     else None
 )

 # Initialize our Trainer
 UpperCAmelCase__   :					List[Any]   				=   Trainer(
     model=__UpperCamelCase							,      args=__UpperCamelCase							,      train_dataset=__UpperCamelCase							,      eval_dataset=__UpperCamelCase							,      data_collator=__UpperCamelCase							,      )

 # Training
 if training_args.do_train:
  trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path ) else None )
  trainer.save_model()
  # For convenience, we also re-save the tokenizer to the same directory,
  # so that you can share your model easily on huggingface.co/models =)
  if trainer.is_world_master():
   tokenizer.save_pretrained(training_args.output_dir )

    # Evaluation
 if training_args.do_eval:
  logger.info("""*** Evaluate ***""" )

  UpperCAmelCase__   :					Optional[Any]   				=   trainer.predict(__UpperCamelCase )
  UpperCAmelCase__   :					List[str]   				=   output.predictions
  UpperCAmelCase__   :					Dict   				=   np.argmax(__UpperCamelCase							,      axis=1 )

  UpperCAmelCase__   :					Optional[Any]   				=   [ex.pairID for ex in eval_dataset]
  UpperCAmelCase__   :					Dict   				=   os.path.join(training_args.output_dir							,      """hans_predictions.txt""" )
  UpperCAmelCase__   :					Dict   				=   eval_dataset.get_labels()
  if trainer.is_world_master():
   with open(__UpperCamelCase							,      """w""" ) as writer:
    writer.write("""pairID,gold_label\n""" )
    for pid, pred in zip(__UpperCamelCase							,      __UpperCamelCase ):
     writer.write("""ex""" + str(__UpperCamelCase ) + """,""" + label_list[int(__UpperCamelCase )] + """\n""" )

  trainer._log(output.metrics )
def 					lowerCAmelCase  (   __UpperCamelCase ):
 '''simple docstring'''





 main()


if __name__ == "__main__":
      main()





