




import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


__SCREAMING_SNAKE_CASE      :				Tuple				      =logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
__SCREAMING_SNAKE_CASE      :				List[Any]				      =direct_transformers_import(Path(__file__).parent)


__SCREAMING_SNAKE_CASE      :				Dict				      ={
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}






class      A     (						_lowerCAmelCase						):
     """simple docstring"""

     _snake_case:		Dict       =["feature_extractor", "tokenizer"]
     # Names need to be attr_class for attr in attributes
     _snake_case:		int       =None
     _snake_case:		Union[str, Any]       =None
     _snake_case:		Optional[Any]       =None




     def __init__(      self					:				Union[str, Any]		,					*A__					:				Optional[Any]		,					**A__					:				int     )			->    Optional[Any]:
           # Sanitize args and kwargs
           for key in kwargs:
                 if key not in self.attributes:
                       raise TypeError(F"""Unexpected keyword argument {key}."""     )
           for arg, attribute_name in zip(A__		,					self.attributes     ):
                 if attribute_name in kwargs:
                       raise TypeError(F"""Got multiple values for argument {attribute_name}."""     )
                 else:
                       lowerCamelCase__ 					=			arg

           if len(A__     ) != len(self.attributes     ):
                 raise ValueError(
                     F"""This processor requires {len(self.attributes     )} arguments: {", ".join(self.attributes     )}. Got """
                     F"""{len(A__     )} arguments instead."""     )

           # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
           for attribute_name, arg in kwargs.items():
                 lowerCamelCase__ 					=			getattr(self		,					F"""{attribute_name}_class"""     )
                 # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
                 lowerCamelCase__ 					=			AUTO_TO_BASE_CLASS_MAPPING.get(A__		,					A__     )
                 if isinstance(A__		,					A__     ):
                       lowerCamelCase__ 					=			tuple(getattr(A__		,					A__     ) for n in class_name if n is not None     )
                 else:
                       lowerCamelCase__ 					=			getattr(A__		,					A__     )

                 if not isinstance(A__		,					A__     ):
                       raise ValueError(
                           F"""Received a {type(A__     ).__name__} for argument {attribute_name}, but a {class_name} was expected."""     )

                 setattr(self		,					A__		,					A__     )




     def __repr__(      self					:				Any     )			->    Any:
           lowerCamelCase__ 					=			[F"""- {name}: {repr(getattr(self		,					A__     )     )}""" for name in self.attributes]
           lowerCamelCase__ 					=			'\n'.join(A__     )
           return F"""{self.__class__.__name__}:\n{attributes_repr}"""




     def 	__snake_case				(      self					:				Union[str, Any]		,					A__					:				Optional[Any]		,					A__					:				bool = False		,					**A__					:				Union[str, Any]     )			->    Dict:
           os.makedirs(A__		,					exist_ok=A__     )

           if push_to_hub:
                 lowerCamelCase__ 					=			kwargs.pop('commit_message'		,					A__     )
                 lowerCamelCase__ 					=			kwargs.pop('repo_id'		,					save_directory.split(os.path.sep     )[-1]     )
                 lowerCamelCase__ 					=			self._create_repo(A__		,					**A__     )
                 lowerCamelCase__ 					=			self._get_files_timestamps(A__     )
           # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
           # loaded from the Hub.
           if self._auto_class is not None:
                 lowerCamelCase__ 					=			[getattr(self		,					A__     ) for attribute_name in self.attributes]
                 lowerCamelCase__ 					=			[(a.init_kwargs if isinstance(A__		,					A__     ) else a) for a in attrs]
                 custom_object_save(self		,					A__		,					config=A__     )

           for attribute_name in self.attributes:
                 lowerCamelCase__ 					=			getattr(self		,					A__     )
                 # Include the processor class in the attribute config so this processor can then be reloaded with the
                 # `AutoProcessor` API.
                 if hasattr(A__		,					'_set_processor_class'     ):
                       attribute._set_processor_class(self.__class__.__name__     )
                 attribute.save_pretrained(A__     )

           if self._auto_class is not None:
                 # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
                 for attribute_name in self.attributes:
                       lowerCamelCase__ 					=			getattr(self		,					A__     )
                       if isinstance(A__		,					A__     ):
                             del attribute.init_kwargs["auto_map"]

           if push_to_hub:
                 self._upload_modified_files(
                     A__		,					A__		,					A__		,					commit_message=A__		,					token=kwargs.get('use_auth_token'     )		,					)




     @classmethod
     def 	__snake_case				(      cls					:				List[Any]		,					A__					:				Union[str, os.PathLike]		,					A__					:				Optional[Union[str, os.PathLike]] = None		,					A__					:				bool = False		,					A__					:				bool = False		,					A__					:				Optional[Union[str, bool]] = None		,					A__					:				str = "main"		,					**A__					:				str		,					)			->    Any:
           lowerCamelCase__ 					=			cache_dir
           lowerCamelCase__ 					=			force_download
           lowerCamelCase__ 					=			local_files_only
           lowerCamelCase__ 					=			revision

           lowerCamelCase__ 					=			kwargs.pop('use_auth_token'		,					A__     )
           if use_auth_token is not None:
                 warnings.warn(
                     'The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.'		,					A__     )
                 if token is not None:
                       raise ValueError(
                           '`token` and `use_auth_token` are both specified. Please set only the argument `token`.'     )
                 lowerCamelCase__ 					=			use_auth_token

           if token is not None:
                 # change to `token` in a follow-up PR
                 lowerCamelCase__ 					=			token

           lowerCamelCase__ 					=			cls._get_arguments_from_pretrained(A__		,					**A__     )
           return cls(*A__     )




     @classmethod
     def 	__snake_case				(      cls					:				Optional[int]		,					A__					:				Union[str, Any]="AutoProcessor"     )			->    str:
           if not isinstance(A__		,					A__     ):
                 lowerCamelCase__ 					=			auto_class.__name__

           import transformers.models.auto as auto_module

           if not hasattr(A__		,					A__     ):
                 raise ValueError(F"""{auto_class} is not a valid auto class."""     )

           lowerCamelCase__ 					=			auto_class




     @classmethod
     def 	__snake_case				(      cls					:				List[str]		,					A__					:				int		,					**A__					:				Union[str, Any]     )			->    List[str]:
           lowerCamelCase__ 					=			[]
           for attribute_name in cls.attributes:
                 lowerCamelCase__ 					=			getattr(cls		,					F"""{attribute_name}_class"""     )
                 if isinstance(A__		,					A__     ):
                       lowerCamelCase__ 					=			tuple(getattr(A__		,					A__     ) if n is not None else None for n in class_name     )
                       lowerCamelCase__ 					=			kwargs.get('use_fast'		,					A__     )
                       if use_fast and classes[1] is not None:
                             lowerCamelCase__ 					=			classes[1]
                       else:
                             lowerCamelCase__ 					=			classes[0]
                 else:
                       lowerCamelCase__ 					=			getattr(A__		,					A__     )

                 args.append(attribute_class.from_pretrained(A__		,					**A__     )     )
           return args




     @property
     def 	__snake_case				(      self					:				Any     )			->    str:
           lowerCamelCase__ 					=			getattr(self		,					self.attributes[0]     )
           return getattr(A__		,					'model_input_names'		,					A__     )
__SCREAMING_SNAKE_CASE      :				List[str]				      =copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
      __SCREAMING_SNAKE_CASE      :				int				      =ProcessorMixin.push_to_hub.__doc__.format(
          object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
      )





# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
      import tensorflow as tf






class      A     (						TensorFormatter[Mapping, "tf.Tensor", Mapping]						):
     """simple docstring"""

     def __init__(      self					:				int		,					A__					:				Tuple=None		,					**A__					:				Tuple     )			->    Dict:
           super().__init__(features=A__     )
           lowerCamelCase__ 					=			tf_tensor_kwargs
           import tensorflow as tf  # noqa: import tf at initialization




     def 	__snake_case				(      self					:				List[Any]		,					A__					:				List[str]     )			->    Dict:
           import tensorflow as tf

           if isinstance(A__		,					A__     ) and column:
                 if all(
                     isinstance(A__		,					tf.Tensor     ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column     ):
                       return tf.stack(A__     )
                 elif all(
                     isinstance(A__		,					(tf.Tensor, tf.RaggedTensor)     ) and x.ndim == 1 and x.dtype == column[0].dtype
                     for x in column     ):
                       # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
                       return tf.ragged.stack(A__     )

           return column




     def 	__snake_case				(      self					:				Tuple		,					A__					:				Optional[int]     )			->    int:
           import tensorflow as tf

           if value is None:
                 return value

           lowerCamelCase__ 					=			{}

           if isinstance(A__		,					(np.number, np.ndarray)     ) and np.issubdtype(value.dtype		,					np.integer     ):
                 lowerCamelCase__ 					=			{'dtype': tf.intaa}
           elif isinstance(A__		,					(np.number, np.ndarray)     ) and np.issubdtype(value.dtype		,					np.floating     ):
                 lowerCamelCase__ 					=			{'dtype': tf.floataa}
           elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                 import PIL.Image

                 if isinstance(A__		,					PIL.Image.Image     ):
                       lowerCamelCase__ 					=			np.asarray(A__     )

           return tf.convert_to_tensor(A__		,					**{**default_dtype, **self.tf_tensor_kwargs}     )




     def 	__snake_case				(      self					:				Optional[int]		,					A__					:				Dict     )			->    List[str]:
           import tensorflow as tf

           # support for torch, tf, jax etc.
           if config.TORCH_AVAILABLE and "torch" in sys.modules:
                 import torch

                 if isinstance(A__		,					torch.Tensor     ):
                       return self._tensorize(data_struct.detach().cpu().numpy()[()]     )
           if hasattr(A__		,					'__array__'     ) and not isinstance(A__		,					tf.Tensor     ):
                 lowerCamelCase__ 					=			data_struct.__array__()
           # support for nested types like struct of list of struct
           if isinstance(A__		,					np.ndarray     ):
                 if data_struct.dtype == object:  # tf tensors cannot be instantied from an array of objects
                       return self._consolidate([self.recursive_tensorize(A__     ) for substruct in data_struct]     )
           elif isinstance(A__		,					(list, tuple)     ):
                 return self._consolidate([self.recursive_tensorize(A__     ) for substruct in data_struct]     )
           return self._tensorize(A__     )




     def 	__snake_case				(      self					:				List[Any]		,					A__					:				dict     )			->    List[str]:
           return map_nested(self._recursive_tensorize		,					A__		,					map_list=A__     )




     def 	__snake_case				(      self					:				Union[str, Any]		,					A__					:				pa.Table     )			->    Mapping:
           lowerCamelCase__ 					=			self.numpy_arrow_extractor().extract_row(A__     )
           lowerCamelCase__ 					=			self.python_features_decoder.decode_row(A__     )
           return self.recursive_tensorize(A__     )




     def 	__snake_case				(      self					:				Optional[Any]		,					A__					:				pa.Table     )			->    "tf.Tensor":
           lowerCamelCase__ 					=			self.numpy_arrow_extractor().extract_column(A__     )
           lowerCamelCase__ 					=			self.python_features_decoder.decode_column(A__		,					pa_table.column_names[0]     )
           lowerCamelCase__ 					=			self.recursive_tensorize(A__     )
           lowerCamelCase__ 					=			self._consolidate(A__     )
           return column




     def 	__snake_case				(      self					:				int		,					A__					:				pa.Table     )			->    Mapping:
           lowerCamelCase__ 					=			self.numpy_arrow_extractor().extract_batch(A__     )
           lowerCamelCase__ 					=			self.python_features_decoder.decode_batch(A__     )
           lowerCamelCase__ 					=			self.recursive_tensorize(A__     )
           for column_name in batch:
                 lowerCamelCase__ 					=			self._consolidate(batch[column_name]     )
           return batch