
"""simple docstring"""


import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


lowerCAmelCase								=						logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
lowerCAmelCase								=						direct_transformers_import(Path(__file__).parent)


lowerCAmelCase								=						{
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}

class    A_     (					A__       ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_     				=       ["""feature_extractor""", """tokenizer"""]
							# Names need to be attr_class for attr in attributes
							SCREAMING_SNAKE_CASE_     				=       None
							SCREAMING_SNAKE_CASE_     				=       None
							SCREAMING_SNAKE_CASE_     				=       None





							def __init__(  self  :List[str]					,							*lowerCamelCase_  :List[str]					,							**lowerCamelCase_  :Union[str, Any]	):





										"""simple docstring"""

										for key in kwargs:
													if key not in self.attributes:
																raise TypeError(f"""Unexpected keyword argument {key}."""	)
										for arg, attribute_name in zip(lowerCamelCase_					,							self.attributes	):
													if attribute_name in kwargs:
																raise TypeError(f"""Got multiple values for argument {attribute_name}."""	)
													else:
																lowerCamelCase__     :					Union[str, Any]		=arg

										if len(lowerCamelCase_	) != len(self.attributes	):
													raise ValueError(
													    f"""This processor requires {len(self.attributes	)} arguments: {', '.join(self.attributes	)}. Got """
													    f"""{len(lowerCamelCase_	)} arguments instead."""	)

										# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
										for attribute_name, arg in kwargs.items():
													lowerCamelCase__     :					List[str]		=getattr(self					,							f"""{attribute_name}_class"""	)
													# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
													lowerCamelCase__     :					Optional[Any]		=AUTO_TO_BASE_CLASS_MAPPING.get(lowerCamelCase_					,							lowerCamelCase_	)
													if isinstance(lowerCamelCase_					,							lowerCamelCase_	):
																lowerCamelCase__     :					Tuple		=tuple(getattr(lowerCamelCase_					,							lowerCamelCase_	) for n in class_name if n is not None	)
													else:
																lowerCamelCase__     :					Optional[Any]		=getattr(lowerCamelCase_					,							lowerCamelCase_	)

													if not isinstance(lowerCamelCase_					,							lowerCamelCase_	):
																raise ValueError(
																    f"""Received a {type(lowerCamelCase_	).__name__} for argument {attribute_name}, but a {class_name} was expected."""	)

													setattr(self					,							lowerCamelCase_					,							lowerCamelCase_	)





							def __repr__(  self  :Dict	):





										"""simple docstring"""

										lowerCamelCase__     :					Optional[Any]		=[f"""- {name}: {repr(getattr(self					,							lowerCamelCase_	)	)}""" for name in self.attributes]
										lowerCamelCase__     :					str		='\n'.join(lowerCamelCase_	)
										return f"""{self.__class__.__name__}:\n{attributes_repr}"""





							def 				UpperCAmelCase__							(  self  :Optional[int]					,							lowerCamelCase_  :str					,							lowerCamelCase_  :bool = False					,							**lowerCamelCase_  :List[str]	):





										"""simple docstring"""

										os.makedirs(lowerCamelCase_					,							exist_ok=lowerCamelCase_	)

										if push_to_hub:
													lowerCamelCase__     :					str		=kwargs.pop('commit_message'					,							lowerCamelCase_	)
													lowerCamelCase__     :					List[str]		=kwargs.pop('repo_id'					,							save_directory.split(os.path.sep	)[-1]	)
													lowerCamelCase__     :					Any		=self._create_repo(lowerCamelCase_					,							**lowerCamelCase_	)
													lowerCamelCase__     :					str		=self._get_files_timestamps(lowerCamelCase_	)
										# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
										# loaded from the Hub.
										if self._auto_class is not None:
													lowerCamelCase__     :					Optional[int]		=[getattr(self					,							lowerCamelCase_	) for attribute_name in self.attributes]
													lowerCamelCase__     :					Union[str, Any]		=[(a.init_kwargs if isinstance(lowerCamelCase_					,							lowerCamelCase_	) else a) for a in attrs]
													custom_object_save(self					,							lowerCamelCase_					,							config=lowerCamelCase_	)

										for attribute_name in self.attributes:
													lowerCamelCase__     :					int		=getattr(self					,							lowerCamelCase_	)
													# Include the processor class in the attribute config so this processor can then be reloaded with the
													# `AutoProcessor` API.
													if hasattr(lowerCamelCase_					,							'_set_processor_class'	):
																attribute._set_processor_class(self.__class__.__name__	)
													attribute.save_pretrained(lowerCamelCase_	)

										if self._auto_class is not None:
													# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
													for attribute_name in self.attributes:
																lowerCamelCase__     :					List[str]		=getattr(self					,							lowerCamelCase_	)
																if isinstance(lowerCamelCase_					,							lowerCamelCase_	):
																			del attribute.init_kwargs["auto_map"]

										if push_to_hub:
													self._upload_modified_files(
													    lowerCamelCase_					,							lowerCamelCase_					,							lowerCamelCase_					,							commit_message=lowerCamelCase_					,							token=kwargs.get('use_auth_token'	)					,							)





							@classmethod
							def 				UpperCAmelCase__							(  cls  :int					,							lowerCamelCase_  :Union[str, os.PathLike]					,							lowerCamelCase_  :Optional[Union[str, os.PathLike]] = None					,							lowerCamelCase_  :bool = False					,							lowerCamelCase_  :bool = False					,							lowerCamelCase_  :Optional[Union[str, bool]] = None					,							lowerCamelCase_  :str = "main"					,							**lowerCamelCase_  :Optional[Any]					,							):





										"""simple docstring"""

										lowerCamelCase__     :					Optional[Any]		=cache_dir
										lowerCamelCase__     :					Dict		=force_download
										lowerCamelCase__     :					str		=local_files_only
										lowerCamelCase__     :					List[str]		=revision

										lowerCamelCase__     :					str		=kwargs.pop('use_auth_token'					,							lowerCamelCase_	)
										if use_auth_token is not None:
													warnings.warn(
													    'The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.'					,							lowerCamelCase_	)
													if token is not None:
																raise ValueError(
																    '`token` and `use_auth_token` are both specified. Please set only the argument `token`.'	)
													lowerCamelCase__     :					Tuple		=use_auth_token

										if token is not None:
													# change to `token` in a follow-up PR
													lowerCamelCase__     :					Tuple		=token

										lowerCamelCase__     :					Tuple		=cls._get_arguments_from_pretrained(lowerCamelCase_					,							**lowerCamelCase_	)
										return cls(*lowerCamelCase_	)





							@classmethod
							def 				UpperCAmelCase__							(  cls  :int					,							lowerCamelCase_  :Optional[Any]="AutoProcessor"	):





										"""simple docstring"""

										if not isinstance(lowerCamelCase_					,							lowerCamelCase_	):
													lowerCamelCase__     :					Optional[Any]		=auto_class.__name__

										import transformers.models.auto as auto_module

										if not hasattr(lowerCamelCase_					,							lowerCamelCase_	):
													raise ValueError(f"""{auto_class} is not a valid auto class."""	)

										lowerCamelCase__     :					Optional[int]		=auto_class





							@classmethod
							def 				UpperCAmelCase__							(  cls  :Tuple					,							lowerCamelCase_  :Union[str, Any]					,							**lowerCamelCase_  :List[Any]	):





										"""simple docstring"""

										lowerCamelCase__     :					List[Any]		=[]
										for attribute_name in cls.attributes:
													lowerCamelCase__     :					Dict		=getattr(cls					,							f"""{attribute_name}_class"""	)
													if isinstance(lowerCamelCase_					,							lowerCamelCase_	):
																lowerCamelCase__     :					List[str]		=tuple(getattr(lowerCamelCase_					,							lowerCamelCase_	) if n is not None else None for n in class_name	)
																lowerCamelCase__     :					Optional[Any]		=kwargs.get('use_fast'					,							lowerCamelCase_	)
																if use_fast and classes[1] is not None:
																			lowerCamelCase__     :					List[Any]		=classes[1]
																else:
																			lowerCamelCase__     :					List[Any]		=classes[0]
													else:
																lowerCamelCase__     :					Any		=getattr(lowerCamelCase_					,							lowerCamelCase_	)

													args.append(attribute_class.from_pretrained(lowerCamelCase_					,							**lowerCamelCase_	)	)
										return args





							@property
							def 				UpperCAmelCase__							(  self  :Tuple	):





										"""simple docstring"""

										lowerCamelCase__     :					str		=getattr(self					,							self.attributes[0]	)
										return getattr(lowerCamelCase_					,							'model_input_names'					,							lowerCamelCase_	)






lowerCAmelCase								=						copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
							lowerCAmelCase								=						ProcessorMixin.push_to_hub.__doc__.format(
							    object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
							)

"""simple docstring"""


from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCAmelCase								=						logging.get_logger(__name__)

lowerCAmelCase								=						{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase								=						{
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase								=						{
    """openai/clip-vit-base-patch32""": 77,
}

class    A_     (					A__       ):



							"""simple docstring"""






							SCREAMING_SNAKE_CASE_     				=       VOCAB_FILES_NAMES
							SCREAMING_SNAKE_CASE_     				=       PRETRAINED_VOCAB_FILES_MAP
							SCREAMING_SNAKE_CASE_     				=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							SCREAMING_SNAKE_CASE_     				=       ["""input_ids""", """attention_mask"""]
							SCREAMING_SNAKE_CASE_     				=       CLIPTokenizer





							def __init__(  self  :List[str]					,							lowerCamelCase_  :Tuple=None					,							lowerCamelCase_  :Any=None					,							lowerCamelCase_  :Optional[Any]=None					,							lowerCamelCase_  :List[Any]="<|endoftext|>"					,							lowerCamelCase_  :Tuple="<|startoftext|>"					,							lowerCamelCase_  :Dict="<|endoftext|>"					,							lowerCamelCase_  :str="<|endoftext|>"					,							**lowerCamelCase_  :int					,							):





										"""simple docstring"""

										super().__init__(
										    lowerCamelCase_					,							lowerCamelCase_					,							tokenizer_file=lowerCamelCase_					,							unk_token=lowerCamelCase_					,							bos_token=lowerCamelCase_					,							eos_token=lowerCamelCase_					,							pad_token=lowerCamelCase_					,							**lowerCamelCase_					,							)

										if not isinstance(self.backend_tokenizer.pre_tokenizer					,							pre_tokenizers.Sequence	):
													raise ValueError(
													    'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
													    ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
													    ' to be compatible with this version.The easiest way to do so is'
													    ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
													    ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
													    ' transformers.'	)

										self._wrap_decode_method_backend_tokenizer()





							def 				UpperCAmelCase__							(  self  :Union[str, Any]	):





										"""simple docstring"""

										lowerCamelCase__     :					List[str]		=self.backend_tokenizer.decode

										def new_decode_method(*lowerCamelCase_  :Optional[int]					,							**lowerCamelCase_  :List[Any]	):
													lowerCamelCase__     :					Any		=orig_decode_method(*lowerCamelCase_					,							**lowerCamelCase_	)
													lowerCamelCase__     :					Optional[int]		=text.replace(self.backend_tokenizer.model.end_of_word_suffix					,							' '	).strip()
													return text

										lowerCamelCase__     :					Optional[Any]		=new_decode_method





							def 				UpperCAmelCase__							(  self  :List[str]					,							lowerCamelCase_  :List[int]					,							lowerCamelCase_  :Optional[List[int]] = None	):





										"""simple docstring"""

										lowerCamelCase__     :					Any		=[self.bos_token_id]
										lowerCamelCase__     :					Any		=[self.eos_token_id]

										if token_ids_a is None:
													return bos_token + token_ids_a + eos_token
										return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token





							def 				UpperCAmelCase__							(  self  :List[Any]					,							lowerCamelCase_  :List[int]					,							lowerCamelCase_  :Optional[List[int]] = None	):





										"""simple docstring"""

										lowerCamelCase__     :					List[str]		=[self.bos_token_id]
										lowerCamelCase__     :					Optional[Any]		=[self.eos_token_id]

										if token_ids_a is None:
													return len(bos_token + token_ids_a + eos_token	) * [0]
										return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token	) * [0]





							def 				UpperCAmelCase__							(  self  :Tuple					,							lowerCamelCase_  :str					,							lowerCamelCase_  :Optional[str] = None	):





										"""simple docstring"""

										lowerCamelCase__     :					int		=self._tokenizer.model.save(lowerCamelCase_					,							name=lowerCamelCase_	)
										return tuple(lowerCamelCase_	)