
import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


lowercase_											=	logging.get_logger(__name__)

lowercase_											=	OrderedDict(
    [
        ("""align""", """AlignProcessor"""),
        ("""altclip""", """AltCLIPProcessor"""),
        ("""bark""", """BarkProcessor"""),
        ("""blip""", """BlipProcessor"""),
        ("""blip-2""", """Blip2Processor"""),
        ("""bridgetower""", """BridgeTowerProcessor"""),
        ("""chinese_clip""", """ChineseCLIPProcessor"""),
        ("""clap""", """ClapProcessor"""),
        ("""clip""", """CLIPProcessor"""),
        ("""clipseg""", """CLIPSegProcessor"""),
        ("""flava""", """FlavaProcessor"""),
        ("""git""", """GitProcessor"""),
        ("""groupvit""", """CLIPProcessor"""),
        ("""hubert""", """Wav2Vec2Processor"""),
        ("""instructblip""", """InstructBlipProcessor"""),
        ("""layoutlmv2""", """LayoutLMv2Processor"""),
        ("""layoutlmv3""", """LayoutLMv3Processor"""),
        ("""markuplm""", """MarkupLMProcessor"""),
        ("""mctct""", """MCTCTProcessor"""),
        ("""mgp-str""", """MgpstrProcessor"""),
        ("""oneformer""", """OneFormerProcessor"""),
        ("""owlvit""", """OwlViTProcessor"""),
        ("""pix2struct""", """Pix2StructProcessor"""),
        ("""sam""", """SamProcessor"""),
        ("""sew""", """Wav2Vec2Processor"""),
        ("""sew-d""", """Wav2Vec2Processor"""),
        ("""speech_to_text""", """Speech2TextProcessor"""),
        ("""speech_to_text_2""", """Speech2Text2Processor"""),
        ("""speecht5""", """SpeechT5Processor"""),
        ("""trocr""", """TrOCRProcessor"""),
        ("""tvlt""", """TvltProcessor"""),
        ("""unispeech""", """Wav2Vec2Processor"""),
        ("""unispeech-sat""", """Wav2Vec2Processor"""),
        ("""vilt""", """ViltProcessor"""),
        ("""vision-text-dual-encoder""", """VisionTextDualEncoderProcessor"""),
        ("""wav2vec2""", """Wav2Vec2Processor"""),
        ("""wav2vec2-conformer""", """Wav2Vec2Processor"""),
        ("""wavlm""", """Wav2Vec2Processor"""),
        ("""whisper""", """WhisperProcessor"""),
        ("""xclip""", """XCLIPProcessor"""),
    ]
)

lowercase_											=	_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)


def     a__			(    snake_case    ):





  """simple docstring"""
  for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
    if class_name in processors:
      __SCREAMING_SNAKE_CASE     :					List[str]			=						model_type_to_module_name(snake_case    )

      __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						importlib.import_module(F'''.{module_name}'''   ,	'''transformers.models'''    )
      try:
        return getattr(snake_case   ,	snake_case    )
      except AttributeError:
        continue

  for processor in PROCESSOR_MAPPING._extra_content.values():
    if getattr(snake_case   ,	'''__name__'''   ,	snake_case    ) == class_name:
      return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						importlib.import_module('''transformers'''    )
  if hasattr(snake_case   ,	snake_case    ):
    return getattr(snake_case   ,	snake_case    )

  return None





class   __UpperCamelCase				:






     """simple docstring"""







     def __init__(							self    :    Tuple				):







       """simple docstring"""




       raise EnvironmentError(
           '''AutoProcessor is designed to be instantiated '''
           '''using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method.'''				)



     @classmethod
     @replace_list_option_in_docstrings(_A				)
     def 				UpperCAmelCase__    (							cls    :    str      ,		_A    :    Union[str, Any]      ,		**_A    :    Optional[int]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Dict			=						kwargs.pop('''config'''      ,		_A				)
       __SCREAMING_SNAKE_CASE     :					Tuple			=						kwargs.pop('''trust_remote_code'''      ,		_A				)
       __SCREAMING_SNAKE_CASE     :					List[str]			=						True

       __SCREAMING_SNAKE_CASE     :					List[Any]			=						None
       __SCREAMING_SNAKE_CASE     :					Dict			=						None

       # First, let's see if we have a preprocessor config.
       # Filter the kwargs for `get_file_from_repo`.
       __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						{
           key: kwargs[key] for key in inspect.signature(_A				).parameters.keys() if key in kwargs
       }
       # Let's start by checking whether the processor class is saved in an image processor
       __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						get_file_from_repo(
           _A      ,		_A      ,		**_A				)
       if preprocessor_config_file is not None:
         __SCREAMING_SNAKE_CASE     :					List[str]			=						ImageProcessingMixin.get_image_processor_dict(_A      ,		**_A				)
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						config_dict.get('''processor_class'''      ,		_A				)
         if "AutoProcessor" in config_dict.get('''auto_map'''      ,		{}				):
           __SCREAMING_SNAKE_CASE     :					Dict			=						config_dict['''auto_map''']['''AutoProcessor''']

        # If not found, let's check whether the processor class is saved in a feature extractor config
       if preprocessor_config_file is not None and processor_class is None:
         __SCREAMING_SNAKE_CASE     :					Optional[int]			=						FeatureExtractionMixin.get_feature_extractor_dict(_A      ,		**_A				)
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						config_dict.get('''processor_class'''      ,		_A				)
         if "AutoProcessor" in config_dict.get('''auto_map'''      ,		{}				):
           __SCREAMING_SNAKE_CASE     :					List[Any]			=						config_dict['''auto_map''']['''AutoProcessor''']

       if processor_class is None:
         # Next, let's check whether the processor class is saved in a tokenizer
         __SCREAMING_SNAKE_CASE     :					Optional[int]			=						get_file_from_repo(
             _A      ,		_A      ,		**_A				)
         if tokenizer_config_file is not None:
           with open(_A      ,		encoding='''utf-8'''				) as reader:
             __SCREAMING_SNAKE_CASE     :					Optional[int]			=						json.load(_A				)

           __SCREAMING_SNAKE_CASE     :					List[str]			=						config_dict.get('''processor_class'''      ,		_A				)
           if "AutoProcessor" in config_dict.get('''auto_map'''      ,		{}				):
             __SCREAMING_SNAKE_CASE     :					Dict			=						config_dict['''auto_map''']['''AutoProcessor''']

       if processor_class is None:
         # Otherwise, load config, if it can be loaded.
         if not isinstance(_A      ,		_A				):
           __SCREAMING_SNAKE_CASE     :					List[Any]			=						AutoConfig.from_pretrained(
               _A      ,		trust_remote_code=_A      ,		**_A				)

         # And check if the config contains the processor class.
         __SCREAMING_SNAKE_CASE     :					Tuple			=						getattr(_A      ,		'''processor_class'''      ,		_A				)
         if hasattr(_A      ,		'''auto_map'''				) and "AutoProcessor" in config.auto_map:
           __SCREAMING_SNAKE_CASE     :					Optional[int]			=						config.auto_map['''AutoProcessor''']

       if processor_class is not None:
         __SCREAMING_SNAKE_CASE     :					List[Any]			=						processor_class_from_name(_A				)

       __SCREAMING_SNAKE_CASE     :					str			=						processor_auto_map is not None
       __SCREAMING_SNAKE_CASE     :					Tuple			=						processor_class is not None or type(_A				) in PROCESSOR_MAPPING
       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						resolve_trust_remote_code(
           _A      ,		_A      ,		_A      ,		_A				)

       if has_remote_code and trust_remote_code:
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						get_class_from_dynamic_module(
             _A      ,		_A      ,		**_A				)
         __SCREAMING_SNAKE_CASE     :					int			=						kwargs.pop('''code_revision'''      ,		_A				)
         if os.path.isdir(_A				):
           processor_class.register_for_auto_class()
         return processor_class.from_pretrained(
             _A      ,		trust_remote_code=_A      ,		**_A				)
       elif processor_class is not None:
         return processor_class.from_pretrained(
             _A      ,		trust_remote_code=_A      ,		**_A				)
       # Last try: we use the PROCESSOR_MAPPING.
       elif type(_A				) in PROCESSOR_MAPPING:
         return PROCESSOR_MAPPING[type(_A				)].from_pretrained(_A      ,		**_A				)

       # At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
       # tokenizer.
       try:
         return AutoTokenizer.from_pretrained(
             _A      ,		trust_remote_code=_A      ,		**_A				)
       except Exception:
         try:
           return AutoImageProcessor.from_pretrained(
               _A      ,		trust_remote_code=_A      ,		**_A				)
         except Exception:
           pass

         try:
           return AutoFeatureExtractor.from_pretrained(
               _A      ,		trust_remote_code=_A      ,		**_A				)
         except Exception:
           pass

       raise ValueError(
           F'''Unrecognized processing class in {pretrained_model_name_or_path}. Can\'t instantiate a processor, a '''
           '''tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains'''
           '''the files of at least one of those processing classes.'''				)




     @staticmethod
     def 				UpperCAmelCase__    (							_A    :    List[str]      ,		_A    :    Tuple				):







       """simple docstring"""




       PROCESSOR_MAPPING.register(_A      ,		_A				)


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowercase_											=	logging.get_logger(__name__)

lowercase_											=	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase_											=	{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

lowercase_											=	{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

lowercase_											=	{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}





class   __UpperCamelCase				(    lowerCAmelCase__   ):






     """simple docstring"""







     lowerCAmelCase_     =	VOCAB_FILES_NAMES
     lowerCAmelCase_     =	PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_     =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_     =	['''input_ids''', '''attention_mask''', '''token_type_ids''']
     lowerCAmelCase_     =	DebertaTokenizer



     def __init__(							self    :    List[str]      ,		_A    :    Tuple=None      ,		_A    :    List[Any]=None      ,		_A    :    Optional[int]=None      ,		_A    :    List[str]="replace"      ,		_A    :    List[Any]="[CLS]"      ,		_A    :    Optional[Any]="[SEP]"      ,		_A    :    Union[str, Any]="[SEP]"      ,		_A    :    List[str]="[CLS]"      ,		_A    :    Optional[Any]="[UNK]"      ,		_A    :    str="[PAD]"      ,		_A    :    List[str]="[MASK]"      ,		_A    :    Dict=False      ,		**_A    :    Optional[int]      ,		):







       """simple docstring"""




       super().__init__(
           _A      ,		_A      ,		tokenizer_file=_A      ,		errors=_A      ,		bos_token=_A      ,		eos_token=_A      ,		unk_token=_A      ,		sep_token=_A      ,		cls_token=_A      ,		pad_token=_A      ,		mask_token=_A      ,		add_prefix_space=_A      ,		**_A      ,		)
       __SCREAMING_SNAKE_CASE     :					List[str]			=						kwargs.pop('''add_bos_token'''      ,		_A				)

       __SCREAMING_SNAKE_CASE     :					int			=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
       if pre_tok_state.get('''add_prefix_space'''      ,		_A				) != add_prefix_space:
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						getattr(_A      ,		pre_tok_state.pop('''type'''				)				)
         __SCREAMING_SNAKE_CASE     :					Dict			=						add_prefix_space
         __SCREAMING_SNAKE_CASE     :					Any			=						pre_tok_class(**_A				)

       __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						add_prefix_space



     @property
     def 				UpperCAmelCase__    (							self    :    List[Any]				):







       """simple docstring"""




       if self._mask_token is None:
         if self.verbose:
           logger.error('''Using mask_token, but it is not set yet.'''				)
         return None
       return str(self._mask_token				)



     @mask_token.setter
     def 				UpperCAmelCase__    (							self    :    Any      ,		_A    :    Union[str, Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Dict			=						AddedToken(_A      ,		lstrip=_A      ,		rstrip=_A				) if isinstance(_A      ,		_A				) else value
       __SCREAMING_SNAKE_CASE     :					Tuple			=						value



     def 				UpperCAmelCase__    (							self    :    List[Any]      ,		_A    :    List[int]      ,		_A    :    Optional[List[int]] = None				):







       """simple docstring"""




       if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       __SCREAMING_SNAKE_CASE     :					List[str]			=						[self.cls_token_id]
       __SCREAMING_SNAKE_CASE     :					List[Any]			=						[self.sep_token_id]
       return cls + token_ids_a + sep + token_ids_a + sep



     def 				UpperCAmelCase__    (							self    :    int      ,		_A    :    List[int]      ,		_A    :    Optional[List[int]] = None				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Tuple			=						[self.sep_token_id]
       __SCREAMING_SNAKE_CASE     :					List[str]			=						[self.cls_token_id]

       if token_ids_a is None:
         return len(cls + token_ids_a + sep				) * [0]
       return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		*_A    :    Optional[Any]      ,		**_A    :    Optional[Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					List[str]			=						kwargs.get('''is_split_into_words'''      ,		_A				)
       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*_A      ,		**_A				)



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		*_A    :    Tuple      ,		**_A    :    int				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Dict			=						kwargs.get('''is_split_into_words'''      ,		_A				)

       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*_A      ,		**_A				)



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    str      ,		_A    :    Optional[str] = None				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						self._tokenizer.model.save(_A      ,		name=_A				)
       return tuple(_A				)




     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    "Conversation"				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					List[Any]			=						[]
       for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(_A      ,		add_special_tokens=_A				) + [self.eos_token_id]				)

       if len(_A				) > self.model_max_length:
         __SCREAMING_SNAKE_CASE     :					Any			=						input_ids[-self.model_max_length :]
       return input_ids
