






"""simple docstring"""


import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


lowercase_    					=					logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
lowercase_    					=					direct_transformers_import(Path(__file__).parent)


lowercase_    					=					{
    'AutoTokenizer': 'PreTrainedTokenizerBase',
    'AutoFeatureExtractor': 'FeatureExtractionMixin',
    'AutoImageProcessor': 'ImageProcessingMixin',
}


class  snake_case		(							_lowerCAmelCase						):


   '''simple docstring'''

   A_						:     Union[str, Any]									=			["feature_extractor", "tokenizer"]
   # Names need to be attr_class for attr in attributes
   A_						:     Optional[int]									=			None
   A_						:     Union[str, Any]									=			None
   A_						:     Any									=			None


   def __init__(     self			:					Any,	*_lowerCamelCase			:					Any,	**_lowerCamelCase			:					List[Any]  ):

     '''simple docstring'''
     # Sanitize args and kwargs
     for key in kwargs:
       if key not in self.attributes:
         raise TypeError(f'Unexpected keyword argument {key}.'  )
     for arg, attribute_name in zip(_lowerCamelCase,	self.attributes  ):
       if attribute_name in kwargs:
         raise TypeError(f'Got multiple values for argument {attribute_name}.'  )
       else:
         __A						      =			arg

     if len(_lowerCamelCase  ) != len(self.attributes  ):
       raise ValueError(
           f'This processor requires {len(self.attributes  )} arguments: {", ".join(self.attributes  )}. Got '
           f'{len(_lowerCamelCase  )} arguments instead.'  )

     # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
     for attribute_name, arg in kwargs.items():
       __A						      =			getattr(self,	f'{attribute_name}_class'  )
       # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
       __A						      =			AUTO_TO_BASE_CLASS_MAPPING.get(_lowerCamelCase,	_lowerCamelCase  )
       if isinstance(_lowerCamelCase,	_lowerCamelCase  ):
         __A						      =			tuple(getattr(_lowerCamelCase,	_lowerCamelCase  ) for n in class_name if n is not None  )
       else:
         __A						      =			getattr(_lowerCamelCase,	_lowerCamelCase  )

       if not isinstance(_lowerCamelCase,	_lowerCamelCase  ):
         raise ValueError(
             f'Received a {type(_lowerCamelCase  ).__name__} for argument {attribute_name}, but a {class_name} was expected.'  )

       setattr(self,	_lowerCamelCase,	_lowerCamelCase  )


   def __repr__(     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			[f'- {name}: {repr(getattr(self,	_lowerCamelCase  )  )}' for name in self.attributes]
     __A						      =			'''\n'''.join(_lowerCamelCase  )
     return f'{self.__class__.__name__}:\n{attributes_repr}'


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any,	_lowerCamelCase			:					Tuple,	_lowerCamelCase			:					bool = False,	**_lowerCamelCase			:					List[Any]  ):

     '''simple docstring'''
     os.makedirs(_lowerCamelCase,	exist_ok=_lowerCamelCase  )

     if push_to_hub:
       __A						      =			kwargs.pop('''commit_message''',	_lowerCamelCase  )
       __A						      =			kwargs.pop('''repo_id''',	save_directory.split(os.path.sep  )[-1]  )
       __A						      =			self._create_repo(_lowerCamelCase,	**_lowerCamelCase  )
       __A						      =			self._get_files_timestamps(_lowerCamelCase  )
     # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
     # loaded from the Hub.
     if self._auto_class is not None:
       __A						      =			[getattr(self,	_lowerCamelCase  ) for attribute_name in self.attributes]
       __A						      =			[(a.init_kwargs if isinstance(_lowerCamelCase,	_lowerCamelCase  ) else a) for a in attrs]
       custom_object_save(self,	_lowerCamelCase,	config=_lowerCamelCase  )

     for attribute_name in self.attributes:
       __A						      =			getattr(self,	_lowerCamelCase  )
       # Include the processor class in the attribute config so this processor can then be reloaded with the
       # `AutoProcessor` API.
       if hasattr(_lowerCamelCase,	'''_set_processor_class'''  ):
         attribute._set_processor_class(self.__class__.__name__  )
       attribute.save_pretrained(_lowerCamelCase  )

     if self._auto_class is not None:
       # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
       for attribute_name in self.attributes:
         __A						      =			getattr(self,	_lowerCamelCase  )
         if isinstance(_lowerCamelCase,	_lowerCamelCase  ):
           del attribute.init_kwargs["auto_map"]

     if push_to_hub:
       self._upload_modified_files(
           _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	commit_message=_lowerCamelCase,	token=kwargs.get('''use_auth_token'''  ),	)


   @classmethod
   def     _SCREAMING_SNAKE_CASE    (     cls			:					Any,	_lowerCamelCase			:					Union[str, os.PathLike],	_lowerCamelCase			:					Optional[Union[str, os.PathLike]] = None,	_lowerCamelCase			:					bool = False,	_lowerCamelCase			:					bool = False,	_lowerCamelCase			:					Optional[Union[str, bool]] = None,	_lowerCamelCase			:					str = "main",	**_lowerCamelCase			:					Union[str, Any],	):

     '''simple docstring'''
     __A						      =			cache_dir
     __A						      =			force_download
     __A						      =			local_files_only
     __A						      =			revision

     __A						      =			kwargs.pop('''use_auth_token''',	_lowerCamelCase  )
     if use_auth_token is not None:
       warnings.warn(
           '''The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.''',	_lowerCamelCase  )
       if token is not None:
         raise ValueError(
             '''`token` and `use_auth_token` are both specified. Please set only the argument `token`.'''  )
       __A						      =			use_auth_token

     if token is not None:
       # change to `token` in a follow-up PR
       __A						      =			token

     __A						      =			cls._get_arguments_from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )
     return cls(*_lowerCamelCase  )


   @classmethod
   def     _SCREAMING_SNAKE_CASE    (     cls			:					List[str],	_lowerCamelCase			:					Union[str, Any]="AutoProcessor"  ):

     '''simple docstring'''
     if not isinstance(_lowerCamelCase,	_lowerCamelCase  ):
       __A						      =			auto_class.__name__

     import transformers.models.auto as auto_module

     if not hasattr(_lowerCamelCase,	_lowerCamelCase  ):
       raise ValueError(f'{auto_class} is not a valid auto class.'  )

     __A						      =			auto_class


   @classmethod
   def     _SCREAMING_SNAKE_CASE    (     cls			:					Union[str, Any],	_lowerCamelCase			:					Optional[Any],	**_lowerCamelCase			:					Union[str, Any]  ):

     '''simple docstring'''
     __A						      =			[]
     for attribute_name in cls.attributes:
       __A						      =			getattr(cls,	f'{attribute_name}_class'  )
       if isinstance(_lowerCamelCase,	_lowerCamelCase  ):
         __A						      =			tuple(getattr(_lowerCamelCase,	_lowerCamelCase  ) if n is not None else None for n in class_name  )
         __A						      =			kwargs.get('''use_fast''',	_lowerCamelCase  )
         if use_fast and classes[1] is not None:
           __A						      =			classes[1]
         else:
           __A						      =			classes[0]
       else:
         __A						      =			getattr(_lowerCamelCase,	_lowerCamelCase  )

       args.append(attribute_class.from_pretrained(_lowerCamelCase,	**_lowerCamelCase  )  )
     return args






   @property
   def     _SCREAMING_SNAKE_CASE    (     self			:					int  ):

     '''simple docstring'''
     __A						      =			getattr(self,	self.attributes[0]  )
     return getattr(_lowerCamelCase,	'''model_input_names''',	_lowerCamelCase  )





lowercase_    					=					copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
       lowercase_    					=					ProcessorMixin.push_to_hub.__doc__.format(
           object='processor', object_class='AutoProcessor', object_files='processor files'
       )












"""simple docstring"""


from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers.models.ctrl.modeling_tf_ctrl import (
           TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
           TFCTRLForSequenceClassification,
           TFCTRLLMHeadModel,
           TFCTRLModel,
       )


class  snake_case		(							_lowerCAmelCase						):


   '''simple docstring'''

   def __init__(     self			:					Optional[int],	_lowerCamelCase			:					Any,	):

     '''simple docstring'''
     __A						      =			parent
     __A						      =			13
     __A						      =			7
     __A						      =			True
     __A						      =			True
     __A						      =			True
     __A						      =			True
     __A						      =			True
     __A						      =			99
     __A						      =			32
     __A						      =			2
     __A						      =			4
     __A						      =			37
     __A						      =			'''gelu'''
     __A						      =			0.1
     __A						      =			0.1
     __A						      =			5_12
     __A						      =			16
     __A						      =			2
     __A						      =			0.02
     __A						      =			3
     __A						      =			4
     __A						      =			None
     __A						      =			self.vocab_size - 1


   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     __A						      =			ids_tensor([self.batch_size, self.seq_length],	self.vocab_size  )

     __A						      =			None
     if self.use_input_mask:
       __A						      =			random_attention_mask([self.batch_size, self.seq_length]  )

     __A						      =			None
     if self.use_token_type_ids:
       __A						      =			ids_tensor([self.batch_size, self.seq_length],	self.type_vocab_size  )

     __A						      =			None
     if self.use_mc_token_ids:
       __A						      =			ids_tensor([self.batch_size, self.num_choices],	self.seq_length  )

     __A						      =			None
     __A						      =			None
     __A						      =			None
     if self.use_labels:
       __A						      =			ids_tensor([self.batch_size],	self.type_sequence_label_size  )
       __A						      =			ids_tensor([self.batch_size, self.seq_length],	self.num_labels  )
       __A						      =			ids_tensor([self.batch_size],	self.num_choices  )

     __A						      =			CTRLConfig(
         vocab_size=self.vocab_size,	n_embd=self.hidden_size,	n_layer=self.num_hidden_layers,	n_head=self.num_attention_heads,	n_positions=self.max_position_embeddings,	pad_token_id=self.pad_token_id,	)

     __A						      =			ids_tensor([self.num_hidden_layers, self.num_attention_heads],	2  )

     return (
         config,
         input_ids,
         input_mask,
         head_mask,
         token_type_ids,
         mc_token_ids,
         sequence_labels,
         token_labels,
         choice_labels,
     )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					Union[str, Any],	_lowerCamelCase			:					int,	_lowerCamelCase			:					List[Any],	*_lowerCamelCase			:					List[Any]  ):

     '''simple docstring'''
     __A						      =			TFCTRLModel(config=_lowerCamelCase  )
     __A						      =			{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A						      =			model(_lowerCamelCase  )

     __A						      =			[input_ids, None, input_mask]  # None is the input for 'past'
     __A						      =			model(_lowerCamelCase  )

     __A						      =			model(_lowerCamelCase  )

     self.parent.assertEqual(result.last_hidden_state.shape,	(self.batch_size, self.seq_length, self.hidden_size)  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int,	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					Optional[Any],	_lowerCamelCase			:					Union[str, Any],	_lowerCamelCase			:					Any,	_lowerCamelCase			:					Optional[Any],	*_lowerCamelCase			:					Tuple  ):

     '''simple docstring'''
     __A						      =			TFCTRLLMHeadModel(config=_lowerCamelCase  )
     __A						      =			{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A						      =			model(_lowerCamelCase  )
     self.parent.assertEqual(result.logits.shape,	(self.batch_size, self.seq_length, self.vocab_size)  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int],	_lowerCamelCase			:					Optional[Any],	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					Tuple,	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					List[str],	*_lowerCamelCase			:					List[str]  ):

     '''simple docstring'''
     __A						      =			self.num_labels
     __A						      =			ids_tensor([self.batch_size],	self.type_sequence_label_size  )
     __A						      =			{
         '''input_ids''': input_ids,
         '''token_type_ids''': token_type_ids,
         '''labels''': sequence_labels,
     }
     __A						      =			TFCTRLForSequenceClassification(_lowerCamelCase  )
     __A						      =			model(_lowerCamelCase  )
     self.parent.assertEqual(result.logits.shape,	(self.batch_size, self.num_labels)  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     __A						      =			self.prepare_config_and_inputs()

     (
         (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   (
         __A
     )      ,   
     )						      =			config_and_inputs

     __A						      =			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict





@require_tf
class  snake_case		(							_lowerCAmelCase       ,     _lowerCAmelCase       ,     unittest.TestCase						):


   '''simple docstring'''

   A_						:     Optional[int]									=			(TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
   A_						:     str									=			(TFCTRLLMHeadModel,) if is_tf_available() else ()
   A_						:     int									=			(
       {
           "feature-extraction": TFCTRLModel,
           "text-classification": TFCTRLForSequenceClassification,
           "text-generation": TFCTRLLMHeadModel,
           "zero-shot": TFCTRLForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   A_						:     Union[str, Any]									=			False
   A_						:     Dict									=			False


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int],	_lowerCamelCase			:					Union[str, Any],	_lowerCamelCase			:					int,	_lowerCamelCase			:					List[str],	_lowerCamelCase			:					str,	_lowerCamelCase			:					str  ):

     '''simple docstring'''
     if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
       # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
       # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
       # config could not be created.
       return True

     return False


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple  ):

     '''simple docstring'''
     __A						      =			TFCTRLModelTester(self  )
     __A						      =			ConfigTester(self,	config_class=_lowerCamelCase,	n_embd=37  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     self.config_tester.run_common_tests()


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any]  ):

     '''simple docstring'''
     __A						      =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_ctrl_model(*_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str]  ):

     '''simple docstring'''
     __A						      =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_ctrl_lm_head(*_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A						      =			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_ctrl_for_sequence_classification(*_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     __A						      =			[TFCTRLLMHeadModel]
     __A						      =			[TFCTRLForSequenceClassification]

     for model_class in self.all_model_classes:
       __A						      =			model_class(_lowerCamelCase  )
       model.build()  # may be needed for the get_bias() call below
       assert isinstance(model.get_input_embeddings(),	tf.keras.layers.Layer  )

       if model_class in list_lm_models:
         __A						      =			model.get_output_embeddings()
         assert isinstance(_lowerCamelCase,	tf.keras.layers.Layer  )
         __A						      =			model.get_bias()
         assert isinstance(_lowerCamelCase,	_lowerCamelCase  )
         for k, v in name.items():
           assert isinstance(_lowerCamelCase,	tf.Variable  )
       elif model_class in list_other_models_with_output_ebd:
         __A						      =			model.get_output_embeddings()
         assert isinstance(_lowerCamelCase,	tf.keras.layers.Layer  )
         __A						      =			model.get_bias()
         assert name is None
       else:
         __A						      =			model.get_output_embeddings()
         assert x is None
         __A						      =			model.get_bias()
         assert name is None






   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       __A						      =			TFCTRLModel.from_pretrained(_lowerCamelCase  )
       self.assertIsNotNone(_lowerCamelCase  )





@require_tf
class  snake_case		(							unittest.TestCase						):


   '''simple docstring'''

   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Dict  ):

     '''simple docstring'''
     __A						      =			TFCTRLLMHeadModel.from_pretrained('''ctrl'''  )
     __A						      =			tf.convert_to_tensor([[1_18_59, 0, 16_11, 8]],	dtype=tf.intaa  )  # Legal the president is
     __A						      =			[
         1_18_59,
         0,
         16_11,
         8,
         5,
         1_50,
         2_64_49,
         2,
         19,
         3_48,
         4_69,
         3,
         25_95,
         48,
         2_07_40,
         24_65_33,
         24_65_33,
         19,
         30,
         5,
     ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

     __A						      =			model.generate(_lowerCamelCase,	do_sample=_lowerCamelCase  )
     self.assertListEqual(output_ids[0].numpy().tolist(),	_lowerCamelCase  )




