






"""simple docstring"""


from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
       import numpy as np
       import tensorflow as tf

       from transformers import (
           TF_MODEL_FOR_CAUSAL_LM_MAPPING,
           TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_MASKED_LM_MAPPING,
           TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
           TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
           TF_MODEL_FOR_PRETRAINING_MAPPING,
           TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
           TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
           TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
           TFSharedEmbeddings,
       )

       if _tf_gpu_memory_limit is not None:
              lowercase_    					=					tf.config.list_physical_devices('GPU')
              for gpu in gpus:
                     # Restrict TensorFlow to only allocate x GB of memory on the GPUs
                     try:
                            tf.config.set_logical_device_configuration(
                                gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
                            )
                            lowercase_    					=					tf.config.list_logical_devices('GPU')
                            print('Logical GPUs', logical_gpus)
                     except RuntimeError as e:
                            # Virtual devices must be set before GPUs have been initialized
                            print(e)


@require_tf
class  snake_case		:


   '''simple docstring'''

   A_						:     Any									=			None
   A_						:     Dict									=			()
   A_						:     Dict									=			()
   A_						:     Tuple									=			True
   A_						:     Any									=			True
   A_						:     Tuple									=			True
   A_						:     int									=			False


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					str=False  ):

     '''simple docstring'''
     __A						      =			copy.deepcopy(_lowerCamelCase  )

     if model_class in get_values(_lowerCamelCase  ):
       __A						      =			{
           k: tf.tile(tf.expand_dims(_lowerCamelCase,	1  ),	(1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)  )
           if isinstance(_lowerCamelCase,	tf.Tensor  ) and v.ndim > 0
           else v
           for k, v in inputs_dict.items()
       }

     if return_labels:
       if model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.ones(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in [
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
       ]:
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in [
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
       ]:
         __A						      =			tf.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length),	dtype=tf.intaa  )
     return inputs_dict


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )

       @tf.function
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )

       @tf.function(experimental_compile=_lowerCamelCase  )
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     # This is a copy of the test_keras_fit method, but we use XLA compilation instead of eager
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			model_class(_lowerCamelCase  )
       if getattr(_lowerCamelCase,	'''hf_compute_loss''',	_lowerCamelCase  ):
         # Test that model correctly compute the loss with kwargs
         __A						      =			self._prepare_for_class(inputs_dict.copy(),	_lowerCamelCase,	return_labels=_lowerCamelCase  )
         # Is there a better way to remove these decoder inputs?
         __A						      =			{
             key: val
             for key, val in prepared_for_class.items()
             if key not in ('''head_mask''', '''decoder_head_mask''', '''cross_attn_head_mask''', '''decoder_input_ids''')
         }

         __A						      =			{
             '''labels''',
             '''label''',
             '''label_ids''',
             '''start_positions''',
             '''start_position''',
             '''end_positions''',
             '''end_position''',
             '''next_sentence_label''',
         }
         __A						      =			possible_label_cols.intersection(set(_lowerCamelCase  )  )
         self.assertGreater(len(_lowerCamelCase  ),	0,	msg='''No matching label names found!'''  )
         __A						      =			{key: val for key, val in prepared_for_class.items() if key in label_names}
         __A						      =			{key: val for key, val in prepared_for_class.items() if key not in label_names}
         self.assertGreater(len(_lowerCamelCase  ),	0  )

         # Make sure it works with XLA!
         model.compile(optimizer=tf.keras.optimizers.SGD(0.0  ),	jit_compile=_lowerCamelCase  )
         # Make sure the model fits without crashing regardless of where we pass the labels
         __A						      =			model.fit(
             _lowerCamelCase,	validation_data=_lowerCamelCase,	steps_per_epoch=1,	validation_steps=1,	shuffle=_lowerCamelCase,	verbose=0,	)
         __A						      =			history.history['''loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )
         __A						      =			history.history['''val_loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )

         # Now test it with separate labels, to make sure that path works in XLA too.
         __A						      =			model_class(_lowerCamelCase  )
         model.compile(optimizer=tf.keras.optimizers.SGD(0.0  ),	jit_compile=_lowerCamelCase  )
         __A						      =			model.fit(
             _lowerCamelCase,	_lowerCamelCase,	validation_data=(inputs_minus_labels, labels),	steps_per_epoch=1,	validation_steps=1,	shuffle=_lowerCamelCase,	verbose=0,	)

         __A						      =			history.history['''loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )
         __A						      =			history.history['''val_loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     __A						      =			True
     __A						      =			True

     if hasattr(_lowerCamelCase,	'''use_cache'''  ):
       __A						      =			True

     __A						      =			getattr(self.model_tester,	'''encoder_seq_length''',	self.model_tester.seq_length  )
     __A						      =			getattr(self.model_tester,	'''key_length''',	_lowerCamelCase  )

     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )
       model.build()
       __A						      =			len(model(_lowerCamelCase  )  )

       for key in list(class_inputs_dict.keys()  ):
         # Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
         if key not in model.input_signature:
           del class_inputs_dict[key]
         # Check it's a tensor, in case the inputs dict has some bools in it too
         elif isinstance(class_inputs_dict[key],	tf.Tensor  ) and class_inputs_dict[key].dtype.is_integer:
           __A						      =			tf.cast(class_inputs_dict[key],	tf.intaa  )

       if set(class_inputs_dict.keys()  ) != set(model.input_signature.keys()  ):
         continue  # Some models have inputs that the preparation functions don't create, we skip those

       with tempfile.TemporaryDirectory() as tmpdirname:
         model.save_pretrained(_lowerCamelCase,	saved_model=_lowerCamelCase  )
         __A						      =			os.path.join(_lowerCamelCase,	'''saved_model''',	'''1'''  )
         __A						      =			tf.keras.models.load_model(_lowerCamelCase  )
         __A						      =			model(_lowerCamelCase  )

         if self.is_encoder_decoder:
           __A						      =			outputs['''encoder_hidden_states''']
           __A						      =			outputs['''encoder_attentions''']
         else:
           __A						      =			outputs['''hidden_states''']
           __A						      =			outputs['''attentions''']

         self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )

         __A						      =			getattr(
             self.model_tester,	'''expected_num_hidden_layers''',	self.model_tester.num_hidden_layers + 1  )

         self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )
         self.assertListEqual(
             list(output_hidden_states[0].shape[-2:]  ),	[self.model_tester.seq_length, self.model_tester.hidden_size],	)

         self.assertEqual(len(_lowerCamelCase  ),	self.model_tester.num_hidden_layers  )
         self.assertListEqual(
             list(output_attentions[0].shape[-3:]  ),	[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],	)


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     tf.keras.mixed_precision.set_global_policy('''mixed_float16'''  )

     # try/finally block to ensure subsequent tests run in float32
     try:
       __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes[:2]:
         __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
         __A						      =			model_class(_lowerCamelCase  )
         __A						      =			model(_lowerCamelCase  )

         self.assertIsNotNone(_lowerCamelCase  )
     finally:
       tf.keras.mixed_precision.set_global_policy('''float32'''  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     # head_mask and decoder_head_mask has different shapes than other input args
     if "head_mask" in inputs_dict:
       del inputs_dict["head_mask"]
     if "decoder_head_mask" in inputs_dict:
       del inputs_dict["decoder_head_mask"]
     if "cross_attn_head_mask" in inputs_dict:
       del inputs_dict["cross_attn_head_mask"]
     __A						      =			{
         module_member
         for model_class in self.all_model_classes
         for module in (import_module(model_class.__module__  ),)
         for module_member_name in dir(_lowerCamelCase  )
         if module_member_name.endswith('''MainLayer'''  )
         for module_member in (getattr(_lowerCamelCase,	_lowerCamelCase  ),)
         if isinstance(_lowerCamelCase,	_lowerCamelCase  )
         and tf.keras.layers.Layer in module_member.__bases__
         and getattr(_lowerCamelCase,	'''_keras_serializable''',	_lowerCamelCase  )
     }

     for main_layer_class in tf_main_layer_classes:
       # T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
       if "T5" in main_layer_class.__name__:
         # Take the same values than in TFT5ModelTester for this shared layer
         __A						      =			TFSharedEmbeddings(self.model_tester.vocab_size,	self.model_tester.hidden_size,	name='''shared'''  )
         __A						      =			False
         __A						      =			main_layer_class(_lowerCamelCase,	embed_tokens=_lowerCamelCase  )
       else:
         __A						      =			main_layer_class(_lowerCamelCase  )

       __A						      =			{
           name: tf.keras.Input(tensor.shape[1:],	dtype=tensor.dtype  ) for name, tensor in inputs_dict.items()
       }

       if hasattr(self.model_tester,	'''num_labels'''  ):
         __A						      =			self.model_tester.num_labels
       else:
         __A						      =			2

       __A						      =			tf.data.Dataset.from_tensor_slices(
           (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)  ))  ).batch(1  )

       __A						      =			main_layer(_lowerCamelCase  )[0]
       __A						      =			tf.keras.layers.Dense(_lowerCamelCase,	activation='''softmax''',	name='''outputs'''  )(_lowerCamelCase  )
       __A						      =			tf.keras.models.Model(inputs=_lowerCamelCase,	outputs=[outputs]  )

       model.compile(loss='''binary_crossentropy''',	optimizer='''adam''',	metrics=['''binary_accuracy''']  )
       model.fit(_lowerCamelCase,	epochs=1  )

       with tempfile.TemporaryDirectory() as tmpdirname:
         __A						      =			os.path.join(_lowerCamelCase,	'''keras_model.h5'''  )
         model.save(_lowerCamelCase  )
         if "T5" in main_layer_class.__name__:
           __A						      =			tf.keras.models.load_model(
               _lowerCamelCase,	custom_objects={
                   main_layer_class.__name__: main_layer_class,
                   '''TFSharedEmbeddings''': TFSharedEmbeddings,
               },	)
         else:
           __A						      =			tf.keras.models.load_model(
               _lowerCamelCase,	custom_objects={main_layer_class.__name__: main_layer_class}  )
         assert isinstance(_lowerCamelCase,	tf.keras.Model  )
         model(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes[:2]:
       __A						      =			model_class(_lowerCamelCase  )

       __A						      =			copy.deepcopy(_lowerCamelCase  )

       if not self.is_encoder_decoder:
         __A						      =			inputs['''input_ids''']
         del inputs["input_ids"]
       else:
         __A						      =			inputs['''input_ids''']
         __A						      =			inputs.get('''decoder_input_ids''',	_lowerCamelCase  )
         del inputs["input_ids"]
         inputs.pop('''decoder_input_ids''',	_lowerCamelCase  )

       if not self.is_encoder_decoder:
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )
       else:
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )

       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )

       @tf.function
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int,	_lowerCamelCase			:					Any,	_lowerCamelCase			:					Union[str, Any]  ):

     '''simple docstring'''
     # special tokens cannot be bad tokens
     __A						      =			[]
     if model.config.bos_token_id is not None:
       special_tokens.append(model.config.bos_token_id  )
     if model.config.pad_token_id is not None:
       special_tokens.append(model.config.pad_token_id  )
     if model.config.eos_token_id is not None:
       special_tokens.append(model.config.eos_token_id  )

     # create random bad tokens that are not special tokens
     __A						      =			[]
     while len(_lowerCamelCase  ) < num_bad_tokens:
       __A						      =			tf.squeeze(ids_tensor((1, 1),	self.model_tester.vocab_size  ),	0  ).numpy()[0]
       if token not in special_tokens:
         bad_tokens.append(_lowerCamelCase  )
     return bad_tokens


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any,	_lowerCamelCase			:					Optional[int]  ):

     '''simple docstring'''
     for token_id in output_ids[0].numpy().tolist():
       self.assertGreaterEqual(_lowerCamelCase,	0  )
       self.assertLess(_lowerCamelCase,	self.model_tester.vocab_size  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					List[str]  ):

     '''simple docstring'''
     # for all bad word tokens
     for bad_word_ids in bad_words_ids:
       # for all slices in batch
       for generated_ids_slice in generated_ids:
         # for all word idx
         for i in range(len(_lowerCamelCase  ),	len(_lowerCamelCase  )  ):
           # if tokens match
           if generated_ids_slice[i - len(_lowerCamelCase  ) : i] == bad_word_ids:
             return True
     return False












"""simple docstring"""


from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
       import numpy as np
       import tensorflow as tf

       from transformers import (
           TF_MODEL_FOR_CAUSAL_LM_MAPPING,
           TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_MASKED_LM_MAPPING,
           TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
           TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
           TF_MODEL_FOR_PRETRAINING_MAPPING,
           TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
           TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
           TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
           TFSharedEmbeddings,
       )

       if _tf_gpu_memory_limit is not None:
              lowercase_    					=					tf.config.list_physical_devices('GPU')
              for gpu in gpus:
                     # Restrict TensorFlow to only allocate x GB of memory on the GPUs
                     try:
                            tf.config.set_logical_device_configuration(
                                gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
                            )
                            lowercase_    					=					tf.config.list_logical_devices('GPU')
                            print('Logical GPUs', logical_gpus)
                     except RuntimeError as e:
                            # Virtual devices must be set before GPUs have been initialized
                            print(e)


@require_tf
class  snake_case		:


   '''simple docstring'''

   A_						:     Any									=			None
   A_						:     Dict									=			()
   A_						:     Dict									=			()
   A_						:     Tuple									=			True
   A_						:     Any									=			True
   A_						:     Tuple									=			True
   A_						:     int									=			False


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					str=False  ):

     '''simple docstring'''
     __A						      =			copy.deepcopy(_lowerCamelCase  )

     if model_class in get_values(_lowerCamelCase  ):
       __A						      =			{
           k: tf.tile(tf.expand_dims(_lowerCamelCase,	1  ),	(1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)  )
           if isinstance(_lowerCamelCase,	tf.Tensor  ) and v.ndim > 0
           else v
           for k, v in inputs_dict.items()
       }

     if return_labels:
       if model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.ones(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in [
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
       ]:
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in [
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
       ]:
         __A						      =			tf.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length),	dtype=tf.intaa  )
     return inputs_dict


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )

       @tf.function
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )

       @tf.function(experimental_compile=_lowerCamelCase  )
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     # This is a copy of the test_keras_fit method, but we use XLA compilation instead of eager
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			model_class(_lowerCamelCase  )
       if getattr(_lowerCamelCase,	'''hf_compute_loss''',	_lowerCamelCase  ):
         # Test that model correctly compute the loss with kwargs
         __A						      =			self._prepare_for_class(inputs_dict.copy(),	_lowerCamelCase,	return_labels=_lowerCamelCase  )
         # Is there a better way to remove these decoder inputs?
         __A						      =			{
             key: val
             for key, val in prepared_for_class.items()
             if key not in ('''head_mask''', '''decoder_head_mask''', '''cross_attn_head_mask''', '''decoder_input_ids''')
         }

         __A						      =			{
             '''labels''',
             '''label''',
             '''label_ids''',
             '''start_positions''',
             '''start_position''',
             '''end_positions''',
             '''end_position''',
             '''next_sentence_label''',
         }
         __A						      =			possible_label_cols.intersection(set(_lowerCamelCase  )  )
         self.assertGreater(len(_lowerCamelCase  ),	0,	msg='''No matching label names found!'''  )
         __A						      =			{key: val for key, val in prepared_for_class.items() if key in label_names}
         __A						      =			{key: val for key, val in prepared_for_class.items() if key not in label_names}
         self.assertGreater(len(_lowerCamelCase  ),	0  )

         # Make sure it works with XLA!
         model.compile(optimizer=tf.keras.optimizers.SGD(0.0  ),	jit_compile=_lowerCamelCase  )
         # Make sure the model fits without crashing regardless of where we pass the labels
         __A						      =			model.fit(
             _lowerCamelCase,	validation_data=_lowerCamelCase,	steps_per_epoch=1,	validation_steps=1,	shuffle=_lowerCamelCase,	verbose=0,	)
         __A						      =			history.history['''loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )
         __A						      =			history.history['''val_loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )

         # Now test it with separate labels, to make sure that path works in XLA too.
         __A						      =			model_class(_lowerCamelCase  )
         model.compile(optimizer=tf.keras.optimizers.SGD(0.0  ),	jit_compile=_lowerCamelCase  )
         __A						      =			model.fit(
             _lowerCamelCase,	_lowerCamelCase,	validation_data=(inputs_minus_labels, labels),	steps_per_epoch=1,	validation_steps=1,	shuffle=_lowerCamelCase,	verbose=0,	)

         __A						      =			history.history['''loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )
         __A						      =			history.history['''val_loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     __A						      =			True
     __A						      =			True

     if hasattr(_lowerCamelCase,	'''use_cache'''  ):
       __A						      =			True

     __A						      =			getattr(self.model_tester,	'''encoder_seq_length''',	self.model_tester.seq_length  )
     __A						      =			getattr(self.model_tester,	'''key_length''',	_lowerCamelCase  )

     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )
       model.build()
       __A						      =			len(model(_lowerCamelCase  )  )

       for key in list(class_inputs_dict.keys()  ):
         # Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
         if key not in model.input_signature:
           del class_inputs_dict[key]
         # Check it's a tensor, in case the inputs dict has some bools in it too
         elif isinstance(class_inputs_dict[key],	tf.Tensor  ) and class_inputs_dict[key].dtype.is_integer:
           __A						      =			tf.cast(class_inputs_dict[key],	tf.intaa  )

       if set(class_inputs_dict.keys()  ) != set(model.input_signature.keys()  ):
         continue  # Some models have inputs that the preparation functions don't create, we skip those

       with tempfile.TemporaryDirectory() as tmpdirname:
         model.save_pretrained(_lowerCamelCase,	saved_model=_lowerCamelCase  )
         __A						      =			os.path.join(_lowerCamelCase,	'''saved_model''',	'''1'''  )
         __A						      =			tf.keras.models.load_model(_lowerCamelCase  )
         __A						      =			model(_lowerCamelCase  )

         if self.is_encoder_decoder:
           __A						      =			outputs['''encoder_hidden_states''']
           __A						      =			outputs['''encoder_attentions''']
         else:
           __A						      =			outputs['''hidden_states''']
           __A						      =			outputs['''attentions''']

         self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )

         __A						      =			getattr(
             self.model_tester,	'''expected_num_hidden_layers''',	self.model_tester.num_hidden_layers + 1  )

         self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )
         self.assertListEqual(
             list(output_hidden_states[0].shape[-2:]  ),	[self.model_tester.seq_length, self.model_tester.hidden_size],	)

         self.assertEqual(len(_lowerCamelCase  ),	self.model_tester.num_hidden_layers  )
         self.assertListEqual(
             list(output_attentions[0].shape[-3:]  ),	[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],	)


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     tf.keras.mixed_precision.set_global_policy('''mixed_float16'''  )

     # try/finally block to ensure subsequent tests run in float32
     try:
       __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes[:2]:
         __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
         __A						      =			model_class(_lowerCamelCase  )
         __A						      =			model(_lowerCamelCase  )

         self.assertIsNotNone(_lowerCamelCase  )
     finally:
       tf.keras.mixed_precision.set_global_policy('''float32'''  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     # head_mask and decoder_head_mask has different shapes than other input args
     if "head_mask" in inputs_dict:
       del inputs_dict["head_mask"]
     if "decoder_head_mask" in inputs_dict:
       del inputs_dict["decoder_head_mask"]
     if "cross_attn_head_mask" in inputs_dict:
       del inputs_dict["cross_attn_head_mask"]
     __A						      =			{
         module_member
         for model_class in self.all_model_classes
         for module in (import_module(model_class.__module__  ),)
         for module_member_name in dir(_lowerCamelCase  )
         if module_member_name.endswith('''MainLayer'''  )
         for module_member in (getattr(_lowerCamelCase,	_lowerCamelCase  ),)
         if isinstance(_lowerCamelCase,	_lowerCamelCase  )
         and tf.keras.layers.Layer in module_member.__bases__
         and getattr(_lowerCamelCase,	'''_keras_serializable''',	_lowerCamelCase  )
     }

     for main_layer_class in tf_main_layer_classes:
       # T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
       if "T5" in main_layer_class.__name__:
         # Take the same values than in TFT5ModelTester for this shared layer
         __A						      =			TFSharedEmbeddings(self.model_tester.vocab_size,	self.model_tester.hidden_size,	name='''shared'''  )
         __A						      =			False
         __A						      =			main_layer_class(_lowerCamelCase,	embed_tokens=_lowerCamelCase  )
       else:
         __A						      =			main_layer_class(_lowerCamelCase  )

       __A						      =			{
           name: tf.keras.Input(tensor.shape[1:],	dtype=tensor.dtype  ) for name, tensor in inputs_dict.items()
       }

       if hasattr(self.model_tester,	'''num_labels'''  ):
         __A						      =			self.model_tester.num_labels
       else:
         __A						      =			2

       __A						      =			tf.data.Dataset.from_tensor_slices(
           (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)  ))  ).batch(1  )

       __A						      =			main_layer(_lowerCamelCase  )[0]
       __A						      =			tf.keras.layers.Dense(_lowerCamelCase,	activation='''softmax''',	name='''outputs'''  )(_lowerCamelCase  )
       __A						      =			tf.keras.models.Model(inputs=_lowerCamelCase,	outputs=[outputs]  )

       model.compile(loss='''binary_crossentropy''',	optimizer='''adam''',	metrics=['''binary_accuracy''']  )
       model.fit(_lowerCamelCase,	epochs=1  )

       with tempfile.TemporaryDirectory() as tmpdirname:
         __A						      =			os.path.join(_lowerCamelCase,	'''keras_model.h5'''  )
         model.save(_lowerCamelCase  )
         if "T5" in main_layer_class.__name__:
           __A						      =			tf.keras.models.load_model(
               _lowerCamelCase,	custom_objects={
                   main_layer_class.__name__: main_layer_class,
                   '''TFSharedEmbeddings''': TFSharedEmbeddings,
               },	)
         else:
           __A						      =			tf.keras.models.load_model(
               _lowerCamelCase,	custom_objects={main_layer_class.__name__: main_layer_class}  )
         assert isinstance(_lowerCamelCase,	tf.keras.Model  )
         model(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes[:2]:
       __A						      =			model_class(_lowerCamelCase  )

       __A						      =			copy.deepcopy(_lowerCamelCase  )

       if not self.is_encoder_decoder:
         __A						      =			inputs['''input_ids''']
         del inputs["input_ids"]
       else:
         __A						      =			inputs['''input_ids''']
         __A						      =			inputs.get('''decoder_input_ids''',	_lowerCamelCase  )
         del inputs["input_ids"]
         inputs.pop('''decoder_input_ids''',	_lowerCamelCase  )

       if not self.is_encoder_decoder:
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )
       else:
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )

       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )

       @tf.function
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int,	_lowerCamelCase			:					Any,	_lowerCamelCase			:					Union[str, Any]  ):

     '''simple docstring'''
     # special tokens cannot be bad tokens
     __A						      =			[]
     if model.config.bos_token_id is not None:
       special_tokens.append(model.config.bos_token_id  )
     if model.config.pad_token_id is not None:
       special_tokens.append(model.config.pad_token_id  )
     if model.config.eos_token_id is not None:
       special_tokens.append(model.config.eos_token_id  )

     # create random bad tokens that are not special tokens
     __A						      =			[]
     while len(_lowerCamelCase  ) < num_bad_tokens:
       __A						      =			tf.squeeze(ids_tensor((1, 1),	self.model_tester.vocab_size  ),	0  ).numpy()[0]
       if token not in special_tokens:
         bad_tokens.append(_lowerCamelCase  )
     return bad_tokens


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any,	_lowerCamelCase			:					Optional[int]  ):

     '''simple docstring'''
     for token_id in output_ids[0].numpy().tolist():
       self.assertGreaterEqual(_lowerCamelCase,	0  )
       self.assertLess(_lowerCamelCase,	self.model_tester.vocab_size  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					List[str]  ):

     '''simple docstring'''
     # for all bad word tokens
     for bad_word_ids in bad_words_ids:
       # for all slices in batch
       for generated_ids_slice in generated_ids:
         # for all word idx
         for i in range(len(_lowerCamelCase  ),	len(_lowerCamelCase  )  ):
           # if tokens match
           if generated_ids_slice[i - len(_lowerCamelCase  ) : i] == bad_word_ids:
             return True
     return False




