






"""simple docstring"""


import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, logging, randn_tensor
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin


lowercase_    					=					logging.get_logger(__name__)  # pylint: disable=invalid-name


@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->EulerDiscrete
class  snake_case		(							_lowerCAmelCase						):


   '''simple docstring'''

   A_						:     torch.FloatTensor
   A_						:     Optional[torch.FloatTensor]									=			None





def  lowerCAmelCase    (   __UpperCamelCase		,	__UpperCamelCase=0.999		,	__UpperCamelCase="cosine"		,	):




  """simple docstring"""







  if alpha_transform_type == "cosine":

    def alpha_bar_fn(__UpperCamelCase			):
      return math.cos((t + 0.008) / 1.008 * math.pi / 2			) ** 2

  elif alpha_transform_type == "exp":

    def alpha_bar_fn(__UpperCamelCase			):
      return math.exp(t * -12.0			)

  else:
    raise ValueError(f'Unsupported alpha_tranform_type: {alpha_transform_type}'			)

  __A						      =			[]
  for i in range(__UpperCamelCase			):
    __A						      =			i / num_diffusion_timesteps
    __A						      =			(i + 1) / num_diffusion_timesteps
    betas.append(min(1 - alpha_bar_fn(__UpperCamelCase			) / alpha_bar_fn(__UpperCamelCase			)		,	__UpperCamelCase			)			)
  return torch.tensor(__UpperCamelCase		,	dtype=torch.floataa			)





class  snake_case		(							_lowerCAmelCase       ,     _lowerCAmelCase						):


   '''simple docstring'''

   A_						:     Tuple									=			[e.name for e in KarrasDiffusionSchedulers]
   A_						:     Optional[Any]									=			1


   @register_to_config
   def __init__(     self			:					Optional[int],	_lowerCamelCase			:					int = 10_00,	_lowerCamelCase			:					float = 0.00_01,	_lowerCamelCase			:					float = 0.02,	_lowerCamelCase			:					str = "linear",	_lowerCamelCase			:					Optional[Union[np.ndarray, List[float]]] = None,	_lowerCamelCase			:					str = "epsilon",	_lowerCamelCase			:					str = "linear",	_lowerCamelCase			:					Optional[bool] = False,	_lowerCamelCase			:					str = "linspace",	_lowerCamelCase			:					int = 0,	):

     '''simple docstring'''
     if trained_betas is not None:
       __A						      =			torch.tensor(_lowerCamelCase,	dtype=torch.floataa  )
     elif beta_schedule == "linear":
       __A						      =			torch.linspace(_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	dtype=torch.floataa  )
     elif beta_schedule == "scaled_linear":
       # this schedule is very specific to the latent diffusion model.
       __A						      =			(
           torch.linspace(beta_start**0.5,	beta_end**0.5,	_lowerCamelCase,	dtype=torch.floataa  ) ** 2
       )
     elif beta_schedule == "squaredcos_cap_v2":
       # Glide cosine schedule
       __A						      =			betas_for_alpha_bar(_lowerCamelCase  )
     else:
       raise NotImplementedError(f'{beta_schedule} does is not implemented for {self.__class__}'  )

     __A						      =			1.0 - self.betas
     __A						      =			torch.cumprod(self.alphas,	dim=0  )

     __A						      =			np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5  )
     __A						      =			np.concatenate([sigmas[::-1], [0.0]]  ).astype(np.floataa  )
     __A						      =			torch.from_numpy(_lowerCamelCase  )

     # setable values
     __A						      =			None
     __A						      =			np.linspace(0,	num_train_timesteps - 1,	_lowerCamelCase,	dtype=_lowerCamelCase  )[::-1].copy()
     __A						      =			torch.from_numpy(_lowerCamelCase  )
     __A						      =			False
     __A						      =			use_karras_sigmas


   @property
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     # standard deviation of the initial noise distribution
     if self.config.timestep_spacing in ["linspace", "trailing"]:
       return self.sigmas.max()

     return (self.sigmas.max() ** 2 + 1) ** 0.5


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					torch.FloatTensor,	_lowerCamelCase			:					Union[float, torch.FloatTensor]  ):

     '''simple docstring'''
     if isinstance(_lowerCamelCase,	torch.Tensor  ):
       __A						      =			timestep.to(self.timesteps.device  )
     __A						      =			(self.timesteps == timestep).nonzero().item()
     __A						      =			self.sigmas[step_index]

     __A						      =			sample / ((sigma**2 + 1) ** 0.5)

     __A						      =			True
     return sample


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any],	_lowerCamelCase			:					int,	_lowerCamelCase			:					Union[str, torch.device] = None  ):

     '''simple docstring'''
     __A						      =			num_inference_steps

     # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
     if self.config.timestep_spacing == "linspace":
       __A						      =			np.linspace(0,	self.config.num_train_timesteps - 1,	_lowerCamelCase,	dtype=_lowerCamelCase  )[
           ::-1
       ].copy()
     elif self.config.timestep_spacing == "leading":
       __A						      =			self.config.num_train_timesteps // self.num_inference_steps
       # creates integer timesteps by multiplying by ratio
       # casting to int to avoid issues when num_inference_step is power of 3
       __A						      =			(np.arange(0,	_lowerCamelCase  ) * step_ratio).round()[::-1].copy().astype(_lowerCamelCase  )
       timesteps += self.config.steps_offset
     elif self.config.timestep_spacing == "trailing":
       __A						      =			self.config.num_train_timesteps / self.num_inference_steps
       # creates integer timesteps by multiplying by ratio
       # casting to int to avoid issues when num_inference_step is power of 3
       __A						      =			(np.arange(self.config.num_train_timesteps,	0,	-step_ratio  )).round().copy().astype(_lowerCamelCase  )
       timesteps -= 1
     else:
       raise ValueError(
           f'{self.config.timestep_spacing} is not supported. Please make sure to choose one of \'linspace\', \'leading\' or \'trailing\'.'  )

     __A						      =			np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5  )
     __A						      =			np.log(_lowerCamelCase  )

     if self.config.interpolation_type == "linear":
       __A						      =			np.interp(_lowerCamelCase,	np.arange(0,	len(_lowerCamelCase  )  ),	_lowerCamelCase  )
     elif self.config.interpolation_type == "log_linear":
       __A						      =			torch.linspace(np.log(sigmas[-1]  ),	np.log(sigmas[0]  ),	num_inference_steps + 1  ).exp()
     else:
       raise ValueError(
           f'{self.config.interpolation_type} is not implemented. Please specify interpolation_type to either'
           ''' \'linear\' or \'log_linear\''''  )

     if self.use_karras_sigmas:
       __A						      =			self._convert_to_karras(in_sigmas=_lowerCamelCase,	num_inference_steps=self.num_inference_steps  )
       __A						      =			np.array([self._sigma_to_t(_lowerCamelCase,	_lowerCamelCase  ) for sigma in sigmas]  )

     __A						      =			np.concatenate([sigmas, [0.0]]  ).astype(np.floataa  )
     __A						      =			torch.from_numpy(_lowerCamelCase  ).to(device=_lowerCamelCase  )
     if str(_lowerCamelCase  ).startswith('''mps'''  ):
       # mps does not support float64
       __A						      =			torch.from_numpy(_lowerCamelCase  ).to(_lowerCamelCase,	dtype=torch.floataa  )
     else:
       __A						      =			torch.from_numpy(_lowerCamelCase  ).to(device=_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Tuple,	_lowerCamelCase			:					Any,	_lowerCamelCase			:					List[Any]  ):

     '''simple docstring'''
     # get log sigma
     __A						      =			np.log(_lowerCamelCase  )

     # get distribution
     __A						      =			log_sigma - log_sigmas[:, np.newaxis]

     # get sigmas range
     __A						      =			np.cumsum((dists >= 0),	axis=0  ).argmax(axis=0  ).clip(max=log_sigmas.shape[0] - 2  )
     __A						      =			low_idx + 1

     __A						      =			log_sigmas[low_idx]
     __A						      =			log_sigmas[high_idx]

     # interpolate sigmas
     __A						      =			(low - log_sigma) / (low - high)
     __A						      =			np.clip(_lowerCamelCase,	0,	1  )

     # transform interpolation to time range
     __A						      =			(1 - w) * low_idx + w * high_idx
     __A						      =			t.reshape(sigma.shape  )
     return t


   def     _SCREAMING_SNAKE_CASE    (     self			:					str,	_lowerCamelCase			:					torch.FloatTensor,	_lowerCamelCase			:					int  ):

     '''simple docstring'''
     __A						      =			in_sigmas[-1].item()
     __A						      =			in_sigmas[0].item()

     __A						      =			7.0  # 7.0 is the value used in the paper
     __A						      =			np.linspace(0,	1,	_lowerCamelCase  )
     __A						      =			sigma_min ** (1 / rho)
     __A						      =			sigma_max ** (1 / rho)
     __A						      =			(max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
     return sigmas


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					torch.FloatTensor,	_lowerCamelCase			:					Union[float, torch.FloatTensor],	_lowerCamelCase			:					torch.FloatTensor,	_lowerCamelCase			:					float = 0.0,	_lowerCamelCase			:					float = 0.0,	_lowerCamelCase			:					float = float('''inf'''  ),	_lowerCamelCase			:					float = 1.0,	_lowerCamelCase			:					Optional[torch.Generator] = None,	_lowerCamelCase			:					bool = True,	):

     '''simple docstring'''

     if (
         isinstance(_lowerCamelCase,	_lowerCamelCase  )
         or isinstance(_lowerCamelCase,	torch.IntTensor  )
         or isinstance(_lowerCamelCase,	torch.LongTensor  )
     ):
       raise ValueError(
           (
               '''Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to'''
               ''' `EulerDiscreteScheduler.step()` is not supported. Make sure to pass'''
               ''' one of the `scheduler.timesteps` as a timestep.'''
           ),	)

     if not self.is_scale_input_called:
       logger.warning(
           '''The `scale_model_input` function should be called before `step` to ensure correct denoising. '''
           '''See `StableDiffusionPipeline` for a usage example.'''  )

     if isinstance(_lowerCamelCase,	torch.Tensor  ):
       __A						      =			timestep.to(self.timesteps.device  )

     __A						      =			(self.timesteps == timestep).nonzero().item()
     __A						      =			self.sigmas[step_index]

     __A						      =			min(s_churn / (len(self.sigmas  ) - 1),	2**0.5 - 1  ) if s_tmin <= sigma <= s_tmax else 0.0

     __A						      =			randn_tensor(
         model_output.shape,	dtype=model_output.dtype,	device=model_output.device,	generator=_lowerCamelCase  )

     __A						      =			noise * s_noise
     __A						      =			sigma * (gamma + 1)

     if gamma > 0:
       __A						      =			sample + eps * (sigma_hat**2 - sigma**2) ** 0.5

     # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
     # NOTE: "original_sample" should not be an expected prediction_type but is left in for
     # backwards compatibility
     if self.config.prediction_type == "original_sample" or self.config.prediction_type == "sample":
       __A						      =			model_output
     elif self.config.prediction_type == "epsilon":
       __A						      =			sample - sigma_hat * model_output
     elif self.config.prediction_type == "v_prediction":
       # * c_out + input * c_skip
       __A						      =			model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1))
     else:
       raise ValueError(
           f'prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`'  )

     # 2. Convert to an ODE derivative
     __A						      =			(sample - pred_original_sample) / sigma_hat

     __A						      =			self.sigmas[step_index + 1] - sigma_hat

     __A						      =			sample + derivative * dt

     if not return_dict:
       return (prev_sample,)

     return EulerDiscreteSchedulerOutput(prev_sample=_lowerCamelCase,	pred_original_sample=_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any,	_lowerCamelCase			:					torch.FloatTensor,	_lowerCamelCase			:					torch.FloatTensor,	_lowerCamelCase			:					torch.FloatTensor,	):

     '''simple docstring'''
     # Make sure sigmas and timesteps have the same device and dtype as original_samples
     __A						      =			self.sigmas.to(device=original_samples.device,	dtype=original_samples.dtype  )
     if original_samples.device.type == "mps" and torch.is_floating_point(_lowerCamelCase  ):
       # mps does not support float64
       __A						      =			self.timesteps.to(original_samples.device,	dtype=torch.floataa  )
       __A						      =			timesteps.to(original_samples.device,	dtype=torch.floataa  )
     else:
       __A						      =			self.timesteps.to(original_samples.device  )
       __A						      =			timesteps.to(original_samples.device  )

     __A						      =			[(schedule_timesteps == t).nonzero().item() for t in timesteps]

     __A						      =			sigmas[step_indices].flatten()
     while len(sigma.shape  ) < len(original_samples.shape  ):
       __A						      =			sigma.unsqueeze(-1  )

     __A						      =			original_samples + noise * sigma
     return noisy_samples






   def __len__(     self			:					Tuple  ):

     '''simple docstring'''
     return self.config.num_train_timesteps












"""simple docstring"""


from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
       import numpy as np
       import tensorflow as tf

       from transformers import (
           TF_MODEL_FOR_CAUSAL_LM_MAPPING,
           TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_MASKED_LM_MAPPING,
           TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
           TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
           TF_MODEL_FOR_PRETRAINING_MAPPING,
           TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
           TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
           TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
           TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
           TFSharedEmbeddings,
       )

       if _tf_gpu_memory_limit is not None:
              lowercase_    					=					tf.config.list_physical_devices('GPU')
              for gpu in gpus:
                     # Restrict TensorFlow to only allocate x GB of memory on the GPUs
                     try:
                            tf.config.set_logical_device_configuration(
                                gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
                            )
                            lowercase_    					=					tf.config.list_logical_devices('GPU')
                            print('Logical GPUs', logical_gpus)
                     except RuntimeError as e:
                            # Virtual devices must be set before GPUs have been initialized
                            print(e)


@require_tf
class  snake_case		:


   '''simple docstring'''

   A_						:     Any									=			None
   A_						:     Dict									=			()
   A_						:     Dict									=			()
   A_						:     Tuple									=			True
   A_						:     Any									=			True
   A_						:     Tuple									=			True
   A_						:     int									=			False


   def     _SCREAMING_SNAKE_CASE    (     self			:					List[Any],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					Dict,	_lowerCamelCase			:					str=False  ):

     '''simple docstring'''
     __A						      =			copy.deepcopy(_lowerCamelCase  )

     if model_class in get_values(_lowerCamelCase  ):
       __A						      =			{
           k: tf.tile(tf.expand_dims(_lowerCamelCase,	1  ),	(1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)  )
           if isinstance(_lowerCamelCase,	tf.Tensor  ) and v.ndim > 0
           else v
           for k, v in inputs_dict.items()
       }

     if return_labels:
       if model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.ones(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in [
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
       ]:
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in get_values(_lowerCamelCase  ):
         __A						      =			tf.zeros(self.model_tester.batch_size,	dtype=tf.intaa  )
       elif model_class in [
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
           *get_values(_lowerCamelCase  ),
       ]:
         __A						      =			tf.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length),	dtype=tf.intaa  )
     return inputs_dict


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[Any]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )

       @tf.function
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Union[str, Any]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )

       @tf.function(experimental_compile=_lowerCamelCase  )
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     # This is a copy of the test_keras_fit method, but we use XLA compilation instead of eager
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes[:2]:
       __A						      =			model_class(_lowerCamelCase  )
       if getattr(_lowerCamelCase,	'''hf_compute_loss''',	_lowerCamelCase  ):
         # Test that model correctly compute the loss with kwargs
         __A						      =			self._prepare_for_class(inputs_dict.copy(),	_lowerCamelCase,	return_labels=_lowerCamelCase  )
         # Is there a better way to remove these decoder inputs?
         __A						      =			{
             key: val
             for key, val in prepared_for_class.items()
             if key not in ('''head_mask''', '''decoder_head_mask''', '''cross_attn_head_mask''', '''decoder_input_ids''')
         }

         __A						      =			{
             '''labels''',
             '''label''',
             '''label_ids''',
             '''start_positions''',
             '''start_position''',
             '''end_positions''',
             '''end_position''',
             '''next_sentence_label''',
         }
         __A						      =			possible_label_cols.intersection(set(_lowerCamelCase  )  )
         self.assertGreater(len(_lowerCamelCase  ),	0,	msg='''No matching label names found!'''  )
         __A						      =			{key: val for key, val in prepared_for_class.items() if key in label_names}
         __A						      =			{key: val for key, val in prepared_for_class.items() if key not in label_names}
         self.assertGreater(len(_lowerCamelCase  ),	0  )

         # Make sure it works with XLA!
         model.compile(optimizer=tf.keras.optimizers.SGD(0.0  ),	jit_compile=_lowerCamelCase  )
         # Make sure the model fits without crashing regardless of where we pass the labels
         __A						      =			model.fit(
             _lowerCamelCase,	validation_data=_lowerCamelCase,	steps_per_epoch=1,	validation_steps=1,	shuffle=_lowerCamelCase,	verbose=0,	)
         __A						      =			history.history['''loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )
         __A						      =			history.history['''val_loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )

         # Now test it with separate labels, to make sure that path works in XLA too.
         __A						      =			model_class(_lowerCamelCase  )
         model.compile(optimizer=tf.keras.optimizers.SGD(0.0  ),	jit_compile=_lowerCamelCase  )
         __A						      =			model.fit(
             _lowerCamelCase,	_lowerCamelCase,	validation_data=(inputs_minus_labels, labels),	steps_per_epoch=1,	validation_steps=1,	shuffle=_lowerCamelCase,	verbose=0,	)

         __A						      =			history.history['''loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )
         __A						      =			history.history['''val_loss'''][0]
         self.assertTrue(not isnan(_lowerCamelCase  )  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     __A						      =			True
     __A						      =			True

     if hasattr(_lowerCamelCase,	'''use_cache'''  ):
       __A						      =			True

     __A						      =			getattr(self.model_tester,	'''encoder_seq_length''',	self.model_tester.seq_length  )
     __A						      =			getattr(self.model_tester,	'''key_length''',	_lowerCamelCase  )

     for model_class in self.all_model_classes[:2]:
       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
       __A						      =			model_class(_lowerCamelCase  )
       model.build()
       __A						      =			len(model(_lowerCamelCase  )  )

       for key in list(class_inputs_dict.keys()  ):
         # Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
         if key not in model.input_signature:
           del class_inputs_dict[key]
         # Check it's a tensor, in case the inputs dict has some bools in it too
         elif isinstance(class_inputs_dict[key],	tf.Tensor  ) and class_inputs_dict[key].dtype.is_integer:
           __A						      =			tf.cast(class_inputs_dict[key],	tf.intaa  )

       if set(class_inputs_dict.keys()  ) != set(model.input_signature.keys()  ):
         continue  # Some models have inputs that the preparation functions don't create, we skip those

       with tempfile.TemporaryDirectory() as tmpdirname:
         model.save_pretrained(_lowerCamelCase,	saved_model=_lowerCamelCase  )
         __A						      =			os.path.join(_lowerCamelCase,	'''saved_model''',	'''1'''  )
         __A						      =			tf.keras.models.load_model(_lowerCamelCase  )
         __A						      =			model(_lowerCamelCase  )

         if self.is_encoder_decoder:
           __A						      =			outputs['''encoder_hidden_states''']
           __A						      =			outputs['''encoder_attentions''']
         else:
           __A						      =			outputs['''hidden_states''']
           __A						      =			outputs['''attentions''']

         self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )

         __A						      =			getattr(
             self.model_tester,	'''expected_num_hidden_layers''',	self.model_tester.num_hidden_layers + 1  )

         self.assertEqual(len(_lowerCamelCase  ),	_lowerCamelCase  )
         self.assertListEqual(
             list(output_hidden_states[0].shape[-2:]  ),	[self.model_tester.seq_length, self.model_tester.hidden_size],	)

         self.assertEqual(len(_lowerCamelCase  ),	self.model_tester.num_hidden_layers  )
         self.assertListEqual(
             list(output_attentions[0].shape[-3:]  ),	[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],	)


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					str  ):

     '''simple docstring'''
     tf.keras.mixed_precision.set_global_policy('''mixed_float16'''  )

     # try/finally block to ensure subsequent tests run in float32
     try:
       __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes[:2]:
         __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )
         __A						      =			model_class(_lowerCamelCase  )
         __A						      =			model(_lowerCamelCase  )

         self.assertIsNotNone(_lowerCamelCase  )
     finally:
       tf.keras.mixed_precision.set_global_policy('''float32'''  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()
     # head_mask and decoder_head_mask has different shapes than other input args
     if "head_mask" in inputs_dict:
       del inputs_dict["head_mask"]
     if "decoder_head_mask" in inputs_dict:
       del inputs_dict["decoder_head_mask"]
     if "cross_attn_head_mask" in inputs_dict:
       del inputs_dict["cross_attn_head_mask"]
     __A						      =			{
         module_member
         for model_class in self.all_model_classes
         for module in (import_module(model_class.__module__  ),)
         for module_member_name in dir(_lowerCamelCase  )
         if module_member_name.endswith('''MainLayer'''  )
         for module_member in (getattr(_lowerCamelCase,	_lowerCamelCase  ),)
         if isinstance(_lowerCamelCase,	_lowerCamelCase  )
         and tf.keras.layers.Layer in module_member.__bases__
         and getattr(_lowerCamelCase,	'''_keras_serializable''',	_lowerCamelCase  )
     }

     for main_layer_class in tf_main_layer_classes:
       # T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
       if "T5" in main_layer_class.__name__:
         # Take the same values than in TFT5ModelTester for this shared layer
         __A						      =			TFSharedEmbeddings(self.model_tester.vocab_size,	self.model_tester.hidden_size,	name='''shared'''  )
         __A						      =			False
         __A						      =			main_layer_class(_lowerCamelCase,	embed_tokens=_lowerCamelCase  )
       else:
         __A						      =			main_layer_class(_lowerCamelCase  )

       __A						      =			{
           name: tf.keras.Input(tensor.shape[1:],	dtype=tensor.dtype  ) for name, tensor in inputs_dict.items()
       }

       if hasattr(self.model_tester,	'''num_labels'''  ):
         __A						      =			self.model_tester.num_labels
       else:
         __A						      =			2

       __A						      =			tf.data.Dataset.from_tensor_slices(
           (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)  ))  ).batch(1  )

       __A						      =			main_layer(_lowerCamelCase  )[0]
       __A						      =			tf.keras.layers.Dense(_lowerCamelCase,	activation='''softmax''',	name='''outputs'''  )(_lowerCamelCase  )
       __A						      =			tf.keras.models.Model(inputs=_lowerCamelCase,	outputs=[outputs]  )

       model.compile(loss='''binary_crossentropy''',	optimizer='''adam''',	metrics=['''binary_accuracy''']  )
       model.fit(_lowerCamelCase,	epochs=1  )

       with tempfile.TemporaryDirectory() as tmpdirname:
         __A						      =			os.path.join(_lowerCamelCase,	'''keras_model.h5'''  )
         model.save(_lowerCamelCase  )
         if "T5" in main_layer_class.__name__:
           __A						      =			tf.keras.models.load_model(
               _lowerCamelCase,	custom_objects={
                   main_layer_class.__name__: main_layer_class,
                   '''TFSharedEmbeddings''': TFSharedEmbeddings,
               },	)
         else:
           __A						      =			tf.keras.models.load_model(
               _lowerCamelCase,	custom_objects={main_layer_class.__name__: main_layer_class}  )
         assert isinstance(_lowerCamelCase,	tf.keras.Model  )
         model(_lowerCamelCase  )


   @slow
   def     _SCREAMING_SNAKE_CASE    (     self			:					Optional[int]  ):

     '''simple docstring'''
     __A      ,   __A						      =			self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes[:2]:
       __A						      =			model_class(_lowerCamelCase  )

       __A						      =			copy.deepcopy(_lowerCamelCase  )

       if not self.is_encoder_decoder:
         __A						      =			inputs['''input_ids''']
         del inputs["input_ids"]
       else:
         __A						      =			inputs['''input_ids''']
         __A						      =			inputs.get('''decoder_input_ids''',	_lowerCamelCase  )
         del inputs["input_ids"]
         inputs.pop('''decoder_input_ids''',	_lowerCamelCase  )

       if not self.is_encoder_decoder:
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )
       else:
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )
         __A						      =			model.get_input_embeddings()(_lowerCamelCase  )

       __A						      =			self._prepare_for_class(_lowerCamelCase,	_lowerCamelCase  )

       @tf.function
       def run_in_graph_mode():
         return model(_lowerCamelCase  )

       __A						      =			run_in_graph_mode()
       self.assertIsNotNone(_lowerCamelCase  )


   def     _SCREAMING_SNAKE_CASE    (     self			:					int,	_lowerCamelCase			:					Any,	_lowerCamelCase			:					Union[str, Any]  ):

     '''simple docstring'''
     # special tokens cannot be bad tokens
     __A						      =			[]
     if model.config.bos_token_id is not None:
       special_tokens.append(model.config.bos_token_id  )
     if model.config.pad_token_id is not None:
       special_tokens.append(model.config.pad_token_id  )
     if model.config.eos_token_id is not None:
       special_tokens.append(model.config.eos_token_id  )

     # create random bad tokens that are not special tokens
     __A						      =			[]
     while len(_lowerCamelCase  ) < num_bad_tokens:
       __A						      =			tf.squeeze(ids_tensor((1, 1),	self.model_tester.vocab_size  ),	0  ).numpy()[0]
       if token not in special_tokens:
         bad_tokens.append(_lowerCamelCase  )
     return bad_tokens


   def     _SCREAMING_SNAKE_CASE    (     self			:					Any,	_lowerCamelCase			:					Optional[int]  ):

     '''simple docstring'''
     for token_id in output_ids[0].numpy().tolist():
       self.assertGreaterEqual(_lowerCamelCase,	0  )
       self.assertLess(_lowerCamelCase,	self.model_tester.vocab_size  )






   def     _SCREAMING_SNAKE_CASE    (     self			:					List[str],	_lowerCamelCase			:					Optional[int],	_lowerCamelCase			:					List[str]  ):

     '''simple docstring'''
     # for all bad word tokens
     for bad_word_ids in bad_words_ids:
       # for all slices in batch
       for generated_ids_slice in generated_ids:
         # for all word idx
         for i in range(len(_lowerCamelCase  ),	len(_lowerCamelCase  )  ):
           # if tokens match
           if generated_ids_slice[i - len(_lowerCamelCase  ) : i] == bad_word_ids:
             return True
     return False




