




# Lint as: python3

import hashlib
from typing import Union



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		Union[str, int, bytes]			)					->       bytes:
 if isinstance(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			):
  # Data already in bytes, returns as it as
  return hash_data
 elif isinstance(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			):
  # We keep the data as it as for it ot be later encoded to UTF-8
  # However replace `\\` with `/` for Windows compatibility
  _snake_case      :					str					=   hash_data.replace("""\\"""			,    """/"""			)
 elif isinstance(SCREAMING_SNAKE_CASE__			,    SCREAMING_SNAKE_CASE__			):
  _snake_case      :					List[str]					=   str(SCREAMING_SNAKE_CASE__			)
 else:
  # If data is not of the required type, raise error
  raise InvalidKeyError(SCREAMING_SNAKE_CASE__			)

 return hash_data.encode("""utf-8"""			)







class 						snake_case  (		SCREAMING_SNAKE_CASE_					):



   '''simple docstring'''






   def __init__(   self							:     List[str]			,						lowerCAmelCase							:     Union[str, Any])			->			List[str]:

    """simple docstring"""







    _snake_case      :					Any					=   """\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
    _snake_case      :					List[str]					=   F'''\nFound Key {hash_data} of type {type(lowerCAmelCase)}'''
    _snake_case      :					List[Any]					=   """\nKeys should be either str, int or bytes type"""
    super().__init__(F'''{self.prefix}{self.err_msg}{self.suffix}''')







class 						snake_case  (		SCREAMING_SNAKE_CASE_					):



   '''simple docstring'''






   def __init__(   self							:     List[str]			,						lowerCAmelCase							:     Tuple			,						lowerCAmelCase							:     Optional[int]			,						lowerCAmelCase							:     List[str]="")			->			Tuple:

    """simple docstring"""







    _snake_case      :					List[Any]					=   key
    _snake_case      :					Union[str, Any]					=   duplicate_key_indices
    _snake_case      :					Dict					=   fix_msg
    _snake_case      :					Optional[Any]					=   """Found multiple examples generated with the same key"""
    if len(lowerCAmelCase) <= 20:
     _snake_case      :					str					=   F'''\nThe examples at index {', '.join(lowerCAmelCase)} have the key {key}'''
    else:
     _snake_case      :					int					=   F'''\nThe examples at index {', '.join(duplicate_key_indices[:20])}... ({len(lowerCAmelCase) - 20} more) have the key {key}'''
    _snake_case      :					Any					=   """\n""" + fix_msg if fix_msg else """"""
    super().__init__(F'''{self.prefix}{self.err_msg}{self.suffix}''')







class 						snake_case  :



   '''simple docstring'''






   def __init__(   self							:     Tuple			,						lowerCAmelCase							:     str)			->			int:

    """simple docstring"""







    _snake_case      :					int					=   hashlib.mda(_as_bytes(lowerCAmelCase))


   def        UpperCamelCase_					(   self							:     int			,						lowerCAmelCase							:     Union[str, int, bytes])			->			int:

    """simple docstring"""







    _snake_case      :					Tuple					=   self._split_mda.copy()
    _snake_case      :					List[Any]					=   _as_bytes(lowerCAmelCase)
    mda.update(lowerCAmelCase)
    # Convert to integer with hexadecimal conversion
    return int(mda.hexdigest()			,						16)





import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


a__      =		logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
a__      =		direct_transformers_import(Path(__file__).parent)


a__      =		{
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}







class 						snake_case  (		SCREAMING_SNAKE_CASE_					):



   '''simple docstring'''






   snake_case_   :	List[str]       =						["""feature_extractor""", """tokenizer"""]
   # Names need to be attr_class for attr in attributes
   snake_case_   :	Optional[Any]       =						None
   snake_case_   :	List[str]       =						None
   snake_case_   :	List[str]       =						None

   def __init__(   self							:     int			,						*lowerCAmelCase							:     Any			,						**lowerCAmelCase							:     int)			->			Optional[Any]:

    """simple docstring"""







    for key in kwargs:
     if key not in self.attributes:
      raise TypeError(F'''Unexpected keyword argument {key}.''')
    for arg, attribute_name in zip(lowerCAmelCase			,						self.attributes):
     if attribute_name in kwargs:
      raise TypeError(F'''Got multiple values for argument {attribute_name}.''')
     else:
      _snake_case      :					List[str]					=   arg

    if len(lowerCAmelCase) != len(self.attributes):
     raise ValueError(
         F'''This processor requires {len(self.attributes)} arguments: {', '.join(self.attributes)}. Got '''
         F'''{len(lowerCAmelCase)} arguments instead.''')

    # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
    for attribute_name, arg in kwargs.items():
     _snake_case      :					Optional[Any]					=   getattr(self			,						F'''{attribute_name}_class''')
     # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
     _snake_case      :					List[str]					=   AUTO_TO_BASE_CLASS_MAPPING.get(lowerCAmelCase			,						lowerCAmelCase)
     if isinstance(lowerCAmelCase			,						lowerCAmelCase):
      _snake_case      :					Optional[Any]					=   tuple(getattr(lowerCAmelCase			,						lowerCAmelCase) for n in class_name if n is not None)
     else:
      _snake_case      :					List[Any]					=   getattr(lowerCAmelCase			,						lowerCAmelCase)

     if not isinstance(lowerCAmelCase			,						lowerCAmelCase):
      raise ValueError(
          F'''Received a {type(lowerCAmelCase).__name__} for argument {attribute_name}, but a {class_name} was expected.''')

     setattr(self			,						lowerCAmelCase			,						lowerCAmelCase)

   def __repr__(   self							:     int)			->			Dict:

    """simple docstring"""







    _snake_case      :					Union[str, Any]					=   [F'''- {name}: {repr(getattr(self			,						lowerCAmelCase))}''' for name in self.attributes]
    _snake_case      :					str					=   """\n""".join(lowerCAmelCase)
    return F'''{self.__class__.__name__}:\n{attributes_repr}'''

   def        UpperCamelCase_					(   self							:     Optional[int]			,						lowerCAmelCase							:     Optional[int]			,						lowerCAmelCase							:     bool = False			,						**lowerCAmelCase							:     Union[str, Any])			->			Dict:

    """simple docstring"""







    os.makedirs(lowerCAmelCase			,						exist_ok=lowerCAmelCase)

    if push_to_hub:
     _snake_case      :					str					=   kwargs.pop("""commit_message"""			,						lowerCAmelCase)
     _snake_case      :					Union[str, Any]					=   kwargs.pop("""repo_id"""			,						save_directory.split(os.path.sep)[-1])
     _snake_case      :					Tuple					=   self._create_repo(lowerCAmelCase			,						**lowerCAmelCase)
     _snake_case      :					Any					=   self._get_files_timestamps(lowerCAmelCase)
    # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
    # loaded from the Hub.
    if self._auto_class is not None:
     _snake_case      :					Any					=   [getattr(self			,						lowerCAmelCase) for attribute_name in self.attributes]
     _snake_case      :					Any					=   [(a.init_kwargs if isinstance(lowerCAmelCase			,						lowerCAmelCase) else a) for a in attrs]
     custom_object_save(self			,						lowerCAmelCase			,						config=lowerCAmelCase)

    for attribute_name in self.attributes:
     _snake_case      :					Optional[int]					=   getattr(self			,						lowerCAmelCase)
     # Include the processor class in the attribute config so this processor can then be reloaded with the
     # `AutoProcessor` API.
     if hasattr(lowerCAmelCase			,						"""_set_processor_class"""):
      attribute._set_processor_class(self.__class__.__name__)
     attribute.save_pretrained(lowerCAmelCase)

    if self._auto_class is not None:
     # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
     for attribute_name in self.attributes:
      _snake_case      :					List[str]					=   getattr(self			,						lowerCAmelCase)
      if isinstance(lowerCAmelCase			,						lowerCAmelCase):
       del attribute.init_kwargs["auto_map"]

    if push_to_hub:
     self._upload_modified_files(
         lowerCAmelCase			,						lowerCAmelCase			,						lowerCAmelCase			,						commit_message=lowerCAmelCase			,						token=kwargs.get("""use_auth_token""")			,						)

   @classmethod
   def        UpperCamelCase_					(   cls							:     List[Any]			,						lowerCAmelCase							:     Union[str, os.PathLike]			,						lowerCAmelCase							:     Optional[Union[str, os.PathLike]] = None			,						lowerCAmelCase							:     bool = False			,						lowerCAmelCase							:     bool = False			,						lowerCAmelCase							:     Optional[Union[str, bool]] = None			,						lowerCAmelCase							:     str = "main"			,						**lowerCAmelCase							:     Dict			,						)			->			Tuple:

    """simple docstring"""







    _snake_case      :					Dict					=   cache_dir
    _snake_case      :					Optional[Any]					=   force_download
    _snake_case      :					Dict					=   local_files_only
    _snake_case      :					Any					=   revision

    _snake_case      :					Dict					=   kwargs.pop("""use_auth_token"""			,						lowerCAmelCase)
    if use_auth_token is not None:
     warnings.warn(
         """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""			,						lowerCAmelCase)
     if token is not None:
      raise ValueError(
          """`token` and `use_auth_token` are both specified. Please set only the argument `token`.""")
     _snake_case      :					str					=   use_auth_token

    if token is not None:
     # change to `token` in a follow-up PR
     _snake_case      :					Tuple					=   token

    _snake_case      :					Tuple					=   cls._get_arguments_from_pretrained(lowerCAmelCase			,						**lowerCAmelCase)
    return cls(*lowerCAmelCase)

   @classmethod
   def        UpperCamelCase_					(   cls							:     Optional[Any]			,						lowerCAmelCase							:     str="AutoProcessor")			->			int:

    """simple docstring"""







    if not isinstance(lowerCAmelCase			,						lowerCAmelCase):
     _snake_case      :					List[Any]					=   auto_class.__name__

    import transformers.models.auto as auto_module

    if not hasattr(lowerCAmelCase			,						lowerCAmelCase):
     raise ValueError(F'''{auto_class} is not a valid auto class.''')

    _snake_case      :					List[str]					=   auto_class

   @classmethod
   def        UpperCamelCase_					(   cls							:     List[Any]			,						lowerCAmelCase							:     Any			,						**lowerCAmelCase							:     Union[str, Any])			->			Optional[Any]:

    """simple docstring"""







    _snake_case      :					Optional[Any]					=   []
    for attribute_name in cls.attributes:
     _snake_case      :					Optional[Any]					=   getattr(cls			,						F'''{attribute_name}_class''')
     if isinstance(lowerCAmelCase			,						lowerCAmelCase):
      _snake_case      :					Tuple					=   tuple(getattr(lowerCAmelCase			,						lowerCAmelCase) if n is not None else None for n in class_name)
      _snake_case      :					str					=   kwargs.get("""use_fast"""			,						lowerCAmelCase)
      if use_fast and classes[1] is not None:
       _snake_case      :					Any					=   classes[1]
      else:
       _snake_case      :					Union[str, Any]					=   classes[0]
     else:
      _snake_case      :					Optional[Any]					=   getattr(lowerCAmelCase			,						lowerCAmelCase)

     args.append(attribute_class.from_pretrained(lowerCAmelCase			,						**lowerCAmelCase))
    return args


   @property
   def        UpperCamelCase_					(   self							:     Optional[int])			->			str:

    """simple docstring"""







    _snake_case      :					Tuple					=   getattr(self			,						self.attributes[0])
    return getattr(lowerCAmelCase			,						"""model_input_names"""			,						lowerCAmelCase)
a__      =		copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
  a__      =		ProcessorMixin.push_to_hub.__doc__.format(
      object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
  )
