

"""simple docstring"""




import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


__UpperCAmelCase	=			logging.get_logger(__name__)

__UpperCAmelCase	=			OrderedDict(
    [
        ('align', 'AlignProcessor'),
        ('altclip', 'AltCLIPProcessor'),
        ('bark', 'BarkProcessor'),
        ('blip', 'BlipProcessor'),
        ('blip-2', 'Blip2Processor'),
        ('bridgetower', 'BridgeTowerProcessor'),
        ('chinese_clip', 'ChineseCLIPProcessor'),
        ('clap', 'ClapProcessor'),
        ('clip', 'CLIPProcessor'),
        ('clipseg', 'CLIPSegProcessor'),
        ('flava', 'FlavaProcessor'),
        ('git', 'GitProcessor'),
        ('groupvit', 'CLIPProcessor'),
        ('hubert', 'Wav2Vec2Processor'),
        ('instructblip', 'InstructBlipProcessor'),
        ('layoutlmv2', 'LayoutLMv2Processor'),
        ('layoutlmv3', 'LayoutLMv3Processor'),
        ('markuplm', 'MarkupLMProcessor'),
        ('mctct', 'MCTCTProcessor'),
        ('mgp-str', 'MgpstrProcessor'),
        ('oneformer', 'OneFormerProcessor'),
        ('owlvit', 'OwlViTProcessor'),
        ('pix2struct', 'Pix2StructProcessor'),
        ('sam', 'SamProcessor'),
        ('sew', 'Wav2Vec2Processor'),
        ('sew-d', 'Wav2Vec2Processor'),
        ('speech_to_text', 'Speech2TextProcessor'),
        ('speech_to_text_2', 'Speech2Text2Processor'),
        ('speecht5', 'SpeechT5Processor'),
        ('trocr', 'TrOCRProcessor'),
        ('tvlt', 'TvltProcessor'),
        ('unispeech', 'Wav2Vec2Processor'),
        ('unispeech-sat', 'Wav2Vec2Processor'),
        ('vilt', 'ViltProcessor'),
        ('vision-text-dual-encoder', 'VisionTextDualEncoderProcessor'),
        ('wav2vec2', 'Wav2Vec2Processor'),
        ('wav2vec2-conformer', 'Wav2Vec2Processor'),
        ('wavlm', 'Wav2Vec2Processor'),
        ('whisper', 'WhisperProcessor'),
        ('xclip', 'XCLIPProcessor'),
    ]
)

__UpperCAmelCase	=			_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)
def 					lowerCAmelCase  (   __UpperCamelCase ):
 '''simple docstring'''





 for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
  if class_name in processors:
   UpperCAmelCase__   :					Dict   				=   model_type_to_module_name(__UpperCamelCase )

   UpperCAmelCase__   :					Dict   				=   importlib.import_module(F".{module_name}"							,      """transformers.models""" )
   try:
    return getattr(__UpperCamelCase							,      __UpperCamelCase )
   except AttributeError:
    continue

 for processor in PROCESSOR_MAPPING._extra_content.values():
  if getattr(__UpperCamelCase							,      """__name__"""							,      __UpperCamelCase ) == class_name:
   return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
 UpperCAmelCase__   :					List[str]   				=   importlib.import_module("""transformers""" )
 if hasattr(__UpperCamelCase							,      __UpperCamelCase ):
  return getattr(__UpperCamelCase							,      __UpperCamelCase )

 return None







class        __lowercase     :

  def __init__(					self						:     Optional[Any]      ):






   '''simple docstring'''
   raise EnvironmentError(
       """AutoProcessor is designed to be instantiated """
       """using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method."""      )

  @classmethod
  @replace_list_option_in_docstrings(A      )
  def 				__lowercase    (					cls						:     Dict   ,A						:     int   ,**A						:     Optional[int]      ):






   '''simple docstring'''
   UpperCAmelCase__   :					Optional[Any]   				=   kwargs.pop("""config"""   ,A      )
   UpperCAmelCase__   :					Union[str, Any]   				=   kwargs.pop("""trust_remote_code"""   ,A      )
   UpperCAmelCase__   :					Union[str, Any]   				=   True

   UpperCAmelCase__   :					Any   				=   None
   UpperCAmelCase__   :					Union[str, Any]   				=   None

   # First, let's see if we have a preprocessor config.
   # Filter the kwargs for `get_file_from_repo`.
   UpperCAmelCase__   :					int   				=   {
       key: kwargs[key] for key in inspect.signature(A      ).parameters.keys() if key in kwargs
   }
   # Let's start by checking whether the processor class is saved in an image processor
   UpperCAmelCase__   :					Optional[Any]   				=   get_file_from_repo(
       A   ,A   ,**A      )
   if preprocessor_config_file is not None:
    UpperCAmelCase__				, UpperCAmelCase__   :					str   				=   ImageProcessingMixin.get_image_processor_dict(A   ,**A      )
    UpperCAmelCase__   :					Any   				=   config_dict.get("""processor_class"""   ,A      )
    if "AutoProcessor" in config_dict.get("""auto_map"""   ,{}      ):
     UpperCAmelCase__   :					Optional[Any]   				=   config_dict["""auto_map"""]["""AutoProcessor"""]

        # If not found, let's check whether the processor class is saved in a feature extractor config
   if preprocessor_config_file is not None and processor_class is None:
    UpperCAmelCase__				, UpperCAmelCase__   :					Optional[int]   				=   FeatureExtractionMixin.get_feature_extractor_dict(A   ,**A      )
    UpperCAmelCase__   :					List[Any]   				=   config_dict.get("""processor_class"""   ,A      )
    if "AutoProcessor" in config_dict.get("""auto_map"""   ,{}      ):
     UpperCAmelCase__   :					str   				=   config_dict["""auto_map"""]["""AutoProcessor"""]

   if processor_class is None:
    # Next, let's check whether the processor class is saved in a tokenizer
    UpperCAmelCase__   :					Tuple   				=   get_file_from_repo(
        A   ,A   ,**A      )
    if tokenizer_config_file is not None:
     with open(A   ,encoding="""utf-8"""      ) as reader:
      UpperCAmelCase__   :					List[Any]   				=   json.load(A      )

     UpperCAmelCase__   :					List[Any]   				=   config_dict.get("""processor_class"""   ,A      )
     if "AutoProcessor" in config_dict.get("""auto_map"""   ,{}      ):
      UpperCAmelCase__   :					Optional[int]   				=   config_dict["""auto_map"""]["""AutoProcessor"""]

   if processor_class is None:
    # Otherwise, load config, if it can be loaded.
    if not isinstance(A   ,A      ):
     UpperCAmelCase__   :					Tuple   				=   AutoConfig.from_pretrained(
         A   ,trust_remote_code=A   ,**A      )

    # And check if the config contains the processor class.
    UpperCAmelCase__   :					Tuple   				=   getattr(A   ,"""processor_class"""   ,A      )
    if hasattr(A   ,"""auto_map"""      ) and "AutoProcessor" in config.auto_map:
     UpperCAmelCase__   :					List[str]   				=   config.auto_map["""AutoProcessor"""]

   if processor_class is not None:
    UpperCAmelCase__   :					List[Any]   				=   processor_class_from_name(A      )

   UpperCAmelCase__   :					Optional[int]   				=   processor_auto_map is not None
   UpperCAmelCase__   :					Union[str, Any]   				=   processor_class is not None or type(A      ) in PROCESSOR_MAPPING
   UpperCAmelCase__   :					Optional[Any]   				=   resolve_trust_remote_code(
       A   ,A   ,A   ,A      )

   if has_remote_code and trust_remote_code:
    UpperCAmelCase__   :					str   				=   get_class_from_dynamic_module(
        A   ,A   ,**A      )
    UpperCAmelCase__   :					str   				=   kwargs.pop("""code_revision"""   ,A      )
    if os.path.isdir(A      ):
     processor_class.register_for_auto_class()
    return processor_class.from_pretrained(
        A   ,trust_remote_code=A   ,**A      )
   elif processor_class is not None:
    return processor_class.from_pretrained(
        A   ,trust_remote_code=A   ,**A      )
   # Last try: we use the PROCESSOR_MAPPING.
   elif type(A      ) in PROCESSOR_MAPPING:
    return PROCESSOR_MAPPING[type(A      )].from_pretrained(A   ,**A      )

   # At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
   # tokenizer.
   try:
    return AutoTokenizer.from_pretrained(
        A   ,trust_remote_code=A   ,**A      )
   except Exception:
    try:
     return AutoImageProcessor.from_pretrained(
         A   ,trust_remote_code=A   ,**A      )
    except Exception:
     pass

    try:
     return AutoFeatureExtractor.from_pretrained(
         A   ,trust_remote_code=A   ,**A      )
    except Exception:
     pass

   raise ValueError(
       f"Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a "
       """tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains"""
       """the files of at least one of those processing classes."""      )




  @staticmethod
  def 				__lowercase    (					A						:     Optional[int]   ,A						:     List[str]      ):






   '''simple docstring'''
   PROCESSOR_MAPPING.register(A   ,A      )








"""simple docstring"""




import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_reformer import ReformerTokenizer
else:
      __UpperCAmelCase	=			None


__UpperCAmelCase	=			logging.get_logger(__name__)


__UpperCAmelCase	=			'▁'

__UpperCAmelCase	=			{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase	=			{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    },
    'tokenizer_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'
        )
    },
}

__UpperCAmelCase	=			{
    'google/reformer-crime-and-punishment': 52_4288,
}







class        __lowercase     (      __lowerCamelCase   ):
  snake_case_			    =    VOCAB_FILES_NAMES
  snake_case_			    =    PRETRAINED_VOCAB_FILES_MAP
  snake_case_			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_			    =    ["""input_ids""", """attention_mask"""]
  snake_case_			    =    ReformerTokenizer

  def __init__(					self						:     List[Any]   ,A						:     Dict=None   ,A						:     List[str]=None   ,A						:     Union[str, Any]="</s>"   ,A						:     List[str]="<unk>"   ,A						:     Tuple=[]   ,**A						:     Optional[int]   ,):






   '''simple docstring'''
   super().__init__(
       A   ,tokenizer_file=A   ,eos_token=A   ,unk_token=A   ,additional_special_tokens=A   ,**A   ,)

   UpperCAmelCase__   :					Tuple   				=   vocab_file
   UpperCAmelCase__   :					Optional[Any]   				=   False if not self.vocab_file else True




  def 				__lowercase    (					self						:     Any   ,A						:     str   ,A						:     Optional[str] = None      ):






   '''simple docstring'''
   if not self.can_save_slow_tokenizer:
    raise ValueError(
        """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
        """tokenizer."""      )

   if not os.path.isdir(A      ):
    logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
    return
   UpperCAmelCase__   :					Tuple   				=   os.path.join(
       A   ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

   if os.path.abspath(self.vocab_file      ) != os.path.abspath(A      ):
    copyfile(self.vocab_file   ,A      )

   return (out_vocab_file,)





