


import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, logging, randn_tensor
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin


__lowerCAmelCase     :			str      =logging.get_logger(__name__)  # pylint: disable=invalid-name
@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->EulerDiscrete
class 	_lowercase				(      A__	):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__    :      torch.FloatTensor
				SCREAMING_SNAKE_CASE__    :      Optional[torch.FloatTensor]      =		None




def 	_UpperCamelCase							(				lowercase__   ,	lowercase__=0.999   ,	lowercase__="cosine"   ,	):
						if alpha_transform_type == "cosine":

												def alpha_bar_fn(lowercase__				):
																		return math.cos((t + 0.008) / 1.008 * math.pi / 2				) ** 2

						elif alpha_transform_type == "exp":

												def alpha_bar_fn(lowercase__				):
																		return math.exp(t * -12.0				)

						else:
												raise ValueError(F'''Unsupported alpha_tranform_type: {alpha_transform_type}'''				)

						__SCREAMING_SNAKE_CASE						:					List[str]		      =		[]
						for i in range(lowercase__				):
												__SCREAMING_SNAKE_CASE						:					List[Any]		      =		i / num_diffusion_timesteps
												__SCREAMING_SNAKE_CASE						:					Dict		      =		(i + 1) / num_diffusion_timesteps
												betas.append(min(1 - alpha_bar_fn(lowercase__				) / alpha_bar_fn(lowercase__				)   ,	lowercase__				)				)
						return torch.tensor(lowercase__   ,	dtype=torch.floataa				)
class 	_lowercase				(      A__						,      A__	):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__    :      int      =		[e.name for e in KarrasDiffusionSchedulers]
				SCREAMING_SNAKE_CASE__    :      Optional[int]      =		1


				@register_to_config
				def __init__(	self      :Tuple      ,      lowerCAmelCase__      :int = 1_000      ,      lowerCAmelCase__      :float = 0.0001      ,      lowerCAmelCase__      :float = 0.02      ,      lowerCAmelCase__      :str = "linear"      ,      lowerCAmelCase__      :Optional[Union[np.ndarray, List[float]]] = None      ,      lowerCAmelCase__      :str = "epsilon"      ,      lowerCAmelCase__      :str = "linear"      ,      lowerCAmelCase__      :Optional[bool] = False      ,      lowerCAmelCase__      :str = "linspace"      ,      lowerCAmelCase__      :int = 0      ,      )		->      Optional[int]:
										if trained_betas is not None:
																__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		torch.tensor(lowerCAmelCase__      ,      dtype=torch.floataa     )
										elif beta_schedule == "linear":
																__SCREAMING_SNAKE_CASE						:					str		      =		torch.linspace(lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      dtype=torch.floataa     )
										elif beta_schedule == "scaled_linear":
																# this schedule is very specific to the latent diffusion model.
																__SCREAMING_SNAKE_CASE						:					List[Any]		      =		(
																    torch.linspace(beta_start**0.5      ,      beta_end**0.5      ,      lowerCAmelCase__      ,      dtype=torch.floataa     ) ** 2
																)
										elif beta_schedule == "squaredcos_cap_v2":
																# Glide cosine schedule
																__SCREAMING_SNAKE_CASE						:					Dict		      =		betas_for_alpha_bar(lowerCAmelCase__     )
										else:
																raise NotImplementedError(f'''{beta_schedule} does is not implemented for {self.__class__}'''     )

										__SCREAMING_SNAKE_CASE						:					Dict		      =		1.0 - self.betas
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		torch.cumprod(self.alphas      ,      dim=0     )

										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5     )
										__SCREAMING_SNAKE_CASE						:					Dict		      =		np.concatenate([sigmas[::-1], [0.0]]     ).astype(np.floataa     )
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		torch.from_numpy(lowerCAmelCase__     )

										# setable values
										__SCREAMING_SNAKE_CASE						:					str		      =		None
										__SCREAMING_SNAKE_CASE						:					int		      =		np.linspace(0      ,      num_train_timesteps - 1      ,      lowerCAmelCase__      ,      dtype=lowerCAmelCase__     )[::-1].copy()
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		torch.from_numpy(lowerCAmelCase__     )
										__SCREAMING_SNAKE_CASE						:					str		      =		False
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		use_karras_sigmas


				@property
				def 		__magic_name__(	self      :int     )		->      Union[str, Any]:
										# standard deviation of the initial noise distribution
										if self.config.timestep_spacing in ["linspace", "trailing"]:
																return self.sigmas.max()

										return (self.sigmas.max() ** 2 + 1) ** 0.5


				def 		__magic_name__(	self      :Dict      ,      lowerCAmelCase__      :torch.FloatTensor      ,      lowerCAmelCase__      :Union[float, torch.FloatTensor]     )		->      torch.FloatTensor:
										if isinstance(lowerCAmelCase__      ,      torch.Tensor     ):
																__SCREAMING_SNAKE_CASE						:					Any		      =		timestep.to(self.timesteps.device     )
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		(self.timesteps == timestep).nonzero().item()
										__SCREAMING_SNAKE_CASE						:					Any		      =		self.sigmas[step_index]

										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		sample / ((sigma**2 + 1) ** 0.5)

										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		True
										return sample


				def 		__magic_name__(	self      :Dict      ,      lowerCAmelCase__      :int      ,      lowerCAmelCase__      :Union[str, torch.device] = None     )		->      int:
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		num_inference_steps

										# "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
										if self.config.timestep_spacing == "linspace":
																__SCREAMING_SNAKE_CASE						:					List[Any]		      =		np.linspace(0      ,      self.config.num_train_timesteps - 1      ,      lowerCAmelCase__      ,      dtype=lowerCAmelCase__     )[
																    ::-1
																].copy()
										elif self.config.timestep_spacing == "leading":
																__SCREAMING_SNAKE_CASE						:					List[str]		      =		self.config.num_train_timesteps // self.num_inference_steps
																# creates integer timesteps by multiplying by ratio
																# casting to int to avoid issues when num_inference_step is power of 3
																__SCREAMING_SNAKE_CASE						:					List[str]		      =		(np.arange(0      ,      lowerCAmelCase__     ) * step_ratio).round()[::-1].copy().astype(lowerCAmelCase__     )
																timesteps += self.config.steps_offset
										elif self.config.timestep_spacing == "trailing":
																__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		self.config.num_train_timesteps / self.num_inference_steps
																# creates integer timesteps by multiplying by ratio
																# casting to int to avoid issues when num_inference_step is power of 3
																__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		(np.arange(self.config.num_train_timesteps      ,      0      ,      -step_ratio     )).round().copy().astype(lowerCAmelCase__     )
																timesteps -= 1
										else:
																raise ValueError(
																    f'''{self.config.timestep_spacing} is not supported. Please make sure to choose one of \'linspace\', \'leading\' or \'trailing\'.'''     )

										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5     )
										__SCREAMING_SNAKE_CASE						:					int		      =		np.log(lowerCAmelCase__     )

										if self.config.interpolation_type == "linear":
																__SCREAMING_SNAKE_CASE						:					str		      =		np.interp(lowerCAmelCase__      ,      np.arange(0      ,      len(lowerCAmelCase__     )     )      ,      lowerCAmelCase__     )
										elif self.config.interpolation_type == "log_linear":
																__SCREAMING_SNAKE_CASE						:					Any		      =		torch.linspace(np.log(sigmas[-1]     )      ,      np.log(sigmas[0]     )      ,      num_inference_steps + 1     ).exp()
										else:
																raise ValueError(
																    f'''{self.config.interpolation_type} is not implemented. Please specify interpolation_type to either'''
																    ''' \'linear\' or \'log_linear\''''     )

										if self.use_karras_sigmas:
																__SCREAMING_SNAKE_CASE						:					Tuple		      =		self._convert_to_karras(in_sigmas=lowerCAmelCase__      ,      num_inference_steps=self.num_inference_steps     )
																__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		np.array([self._sigma_to_t(lowerCAmelCase__      ,      lowerCAmelCase__     ) for sigma in sigmas]     )

										__SCREAMING_SNAKE_CASE						:					Tuple		      =		np.concatenate([sigmas, [0.0]]     ).astype(np.floataa     )
										__SCREAMING_SNAKE_CASE						:					Any		      =		torch.from_numpy(lowerCAmelCase__     ).to(device=lowerCAmelCase__     )
										if str(lowerCAmelCase__     ).startswith('''mps'''     ):
																# mps does not support float64
																__SCREAMING_SNAKE_CASE						:					Tuple		      =		torch.from_numpy(lowerCAmelCase__     ).to(lowerCAmelCase__      ,      dtype=torch.floataa     )
										else:
																__SCREAMING_SNAKE_CASE						:					Tuple		      =		torch.from_numpy(lowerCAmelCase__     ).to(device=lowerCAmelCase__     )


				def 		__magic_name__(	self      :Optional[int]      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Union[str, Any]     )		->      int:
										# get log sigma
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		np.log(lowerCAmelCase__     )

										# get distribution
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		log_sigma - log_sigmas[:, np.newaxis]

										# get sigmas range
										__SCREAMING_SNAKE_CASE						:					int		      =		np.cumsum((dists >= 0)      ,      axis=0     ).argmax(axis=0     ).clip(max=log_sigmas.shape[0] - 2     )
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		low_idx + 1

										__SCREAMING_SNAKE_CASE						:					List[str]		      =		log_sigmas[low_idx]
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		log_sigmas[high_idx]

										# interpolate sigmas
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		(low - log_sigma) / (low - high)
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		np.clip(lowerCAmelCase__      ,      0      ,      1     )

										# transform interpolation to time range
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		(1 - w) * low_idx + w * high_idx
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		t.reshape(sigma.shape     )
										return t


				def 		__magic_name__(	self      :Optional[int]      ,      lowerCAmelCase__      :torch.FloatTensor      ,      lowerCAmelCase__      :Union[str, Any]     )		->      torch.FloatTensor:
										__SCREAMING_SNAKE_CASE						:					float		      =		in_sigmas[-1].item()
										__SCREAMING_SNAKE_CASE						:					float		      =		in_sigmas[0].item()

										__SCREAMING_SNAKE_CASE						:					Tuple		      =		7.0  # 7.0 is the value used in the paper
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		np.linspace(0      ,      1      ,      lowerCAmelCase__     )
										__SCREAMING_SNAKE_CASE						:					str		      =		sigma_min ** (1 / rho)
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		sigma_max ** (1 / rho)
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		(max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
										return sigmas


				def 		__magic_name__(	self      :Union[str, Any]      ,      lowerCAmelCase__      :torch.FloatTensor      ,      lowerCAmelCase__      :Union[float, torch.FloatTensor]      ,      lowerCAmelCase__      :torch.FloatTensor      ,      lowerCAmelCase__      :float = 0.0      ,      lowerCAmelCase__      :float = 0.0      ,      lowerCAmelCase__      :float = float('''inf'''     )      ,      lowerCAmelCase__      :float = 1.0      ,      lowerCAmelCase__      :Optional[torch.Generator] = None      ,      lowerCAmelCase__      :bool = True      ,      )		->      Union[EulerDiscreteSchedulerOutput, Tuple]:

										if (
										    isinstance(lowerCAmelCase__      ,      lowerCAmelCase__     )
										    or isinstance(lowerCAmelCase__      ,      torch.IntTensor     )
										    or isinstance(lowerCAmelCase__      ,      torch.LongTensor     )
										):
																raise ValueError(
																    (
																        '''Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to'''
																        ''' `EulerDiscreteScheduler.step()` is not supported. Make sure to pass'''
																        ''' one of the `scheduler.timesteps` as a timestep.'''
																    )      ,      )

										if not self.is_scale_input_called:
																logger.warning(
																    '''The `scale_model_input` function should be called before `step` to ensure correct denoising. '''
																    '''See `StableDiffusionPipeline` for a usage example.'''     )

										if isinstance(lowerCAmelCase__      ,      torch.Tensor     ):
																__SCREAMING_SNAKE_CASE						:					List[str]		      =		timestep.to(self.timesteps.device     )

										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		(self.timesteps == timestep).nonzero().item()
										__SCREAMING_SNAKE_CASE						:					str		      =		self.sigmas[step_index]

										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		min(s_churn / (len(self.sigmas     ) - 1)      ,      2**0.5 - 1     ) if s_tmin <= sigma <= s_tmax else 0.0

										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		randn_tensor(
										    model_output.shape      ,      dtype=model_output.dtype      ,      device=model_output.device      ,      generator=lowerCAmelCase__     )

										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		noise * s_noise
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		sigma * (gamma + 1)

										if gamma > 0:
																__SCREAMING_SNAKE_CASE						:					str		      =		sample + eps * (sigma_hat**2 - sigma**2) ** 0.5

										# 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
										# NOTE: "original_sample" should not be an expected prediction_type but is left in for
										# backwards compatibility
										if self.config.prediction_type == "original_sample" or self.config.prediction_type == "sample":
																__SCREAMING_SNAKE_CASE						:					Any		      =		model_output
										elif self.config.prediction_type == "epsilon":
																__SCREAMING_SNAKE_CASE						:					Any		      =		sample - sigma_hat * model_output
										elif self.config.prediction_type == "v_prediction":
																# * c_out + input * c_skip
																__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1))
										else:
																raise ValueError(
																    f'''prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`'''     )

										# 2. Convert to an ODE derivative
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		(sample - pred_original_sample) / sigma_hat

										__SCREAMING_SNAKE_CASE						:					Tuple		      =		self.sigmas[step_index + 1] - sigma_hat

										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		sample + derivative * dt

										if not return_dict:
																return (prev_sample,)

										return EulerDiscreteSchedulerOutput(prev_sample=lowerCAmelCase__      ,      pred_original_sample=lowerCAmelCase__     )


				def 		__magic_name__(	self      :str      ,      lowerCAmelCase__      :torch.FloatTensor      ,      lowerCAmelCase__      :torch.FloatTensor      ,      lowerCAmelCase__      :torch.FloatTensor      ,      )		->      torch.FloatTensor:
										# Make sure sigmas and timesteps have the same device and dtype as original_samples
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		self.sigmas.to(device=original_samples.device      ,      dtype=original_samples.dtype     )
										if original_samples.device.type == "mps" and torch.is_floating_point(lowerCAmelCase__     ):
																# mps does not support float64
																__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		self.timesteps.to(original_samples.device      ,      dtype=torch.floataa     )
																__SCREAMING_SNAKE_CASE						:					int		      =		timesteps.to(original_samples.device      ,      dtype=torch.floataa     )
										else:
																__SCREAMING_SNAKE_CASE						:					Tuple		      =		self.timesteps.to(original_samples.device     )
																__SCREAMING_SNAKE_CASE						:					Dict		      =		timesteps.to(original_samples.device     )

										__SCREAMING_SNAKE_CASE						:					Dict		      =		[(schedule_timesteps == t).nonzero().item() for t in timesteps]

										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		sigmas[step_indices].flatten()
										while len(sigma.shape     ) < len(original_samples.shape     ):
																__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		sigma.unsqueeze(-1     )

										__SCREAMING_SNAKE_CASE						:					Any		      =		original_samples + noise * sigma
										return noisy_samples



				def __len__(	self      :Optional[Any]     )		->      Union[str, Any]:
										return self.config.num_train_timesteps




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    RemBertConfig,
						    RemBertForCausalLM,
						    RemBertForMaskedLM,
						    RemBertForMultipleChoice,
						    RemBertForQuestionAnswering,
						    RemBertForSequenceClassification,
						    RemBertForTokenClassification,
						    RemBertModel,
						)
						from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class 	_lowercase				:






				'''simple docstring'''



				def __init__(	self      :int      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Optional[Any]=13      ,      lowerCAmelCase__      :int=7      ,      lowerCAmelCase__      :Any=True      ,      lowerCAmelCase__      :Optional[int]=True      ,      lowerCAmelCase__      :Union[str, Any]=True      ,      lowerCAmelCase__      :Union[str, Any]=True      ,      lowerCAmelCase__      :List[str]=99      ,      lowerCAmelCase__      :Tuple=32      ,      lowerCAmelCase__      :List[str]=18      ,      lowerCAmelCase__      :List[Any]=43      ,      lowerCAmelCase__      :int=5      ,      lowerCAmelCase__      :Any=4      ,      lowerCAmelCase__      :List[str]=37      ,      lowerCAmelCase__      :List[Any]="gelu"      ,      lowerCAmelCase__      :Union[str, Any]=0.1      ,      lowerCAmelCase__      :Union[str, Any]=0.1      ,      lowerCAmelCase__      :Optional[int]=512      ,      lowerCAmelCase__      :Union[str, Any]=16      ,      lowerCAmelCase__      :Optional[Any]=2      ,      lowerCAmelCase__      :Any=0.02      ,      lowerCAmelCase__      :Optional[int]=3      ,      lowerCAmelCase__      :str=4      ,      lowerCAmelCase__      :str=None      ,      )		->      Optional[int]:
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		parent
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		batch_size
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		seq_length
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		is_training
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		use_input_mask
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		use_token_type_ids
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		use_labels
										__SCREAMING_SNAKE_CASE						:					int		      =		vocab_size
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		hidden_size
										__SCREAMING_SNAKE_CASE						:					int		      =		input_embedding_size
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		output_embedding_size
										__SCREAMING_SNAKE_CASE						:					int		      =		num_hidden_layers
										__SCREAMING_SNAKE_CASE						:					int		      =		num_attention_heads
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		intermediate_size
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		hidden_act
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		hidden_dropout_prob
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		attention_probs_dropout_prob
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		max_position_embeddings
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		type_vocab_size
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		type_sequence_label_size
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		initializer_range
										__SCREAMING_SNAKE_CASE						:					Any		      =		num_labels
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		num_choices
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		scope


				def 		__magic_name__(	self      :Any     )		->      Dict:
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		ids_tensor([self.batch_size, self.seq_length]      ,      self.vocab_size     )

										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		None
										if self.use_input_mask:
																__SCREAMING_SNAKE_CASE						:					List[str]		      =		random_attention_mask([self.batch_size, self.seq_length]     )

										__SCREAMING_SNAKE_CASE						:					Dict		      =		None
										if self.use_token_type_ids:
																__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		ids_tensor([self.batch_size, self.seq_length]      ,      self.type_vocab_size     )

										__SCREAMING_SNAKE_CASE						:					Any		      =		None
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		None
										__SCREAMING_SNAKE_CASE						:					Dict		      =		None
										if self.use_labels:
																__SCREAMING_SNAKE_CASE						:					str		      =		ids_tensor([self.batch_size]      ,      self.type_sequence_label_size     )
																__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		ids_tensor([self.batch_size, self.seq_length]      ,      self.num_labels     )
																__SCREAMING_SNAKE_CASE						:					Dict		      =		ids_tensor([self.batch_size]      ,      self.num_choices     )

										__SCREAMING_SNAKE_CASE						:					str		      =		RemBertConfig(
										    vocab_size=self.vocab_size      ,      hidden_size=self.hidden_size      ,      input_embedding_size=self.input_embedding_size      ,      output_embedding_size=self.output_embedding_size      ,      num_hidden_layers=self.num_hidden_layers      ,      num_attention_heads=self.num_attention_heads      ,      intermediate_size=self.intermediate_size      ,      hidden_act=self.hidden_act      ,      hidden_dropout_prob=self.hidden_dropout_prob      ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,      max_position_embeddings=self.max_position_embeddings      ,      type_vocab_size=self.type_vocab_size      ,      is_decoder=lowerCAmelCase__      ,      initializer_range=self.initializer_range      ,      )

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


				def 		__magic_name__(	self      :int     )		->      Any:
										(
										    (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      
										)						:					Dict		      =		self.prepare_config_and_inputs()

										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		True
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		ids_tensor([self.batch_size, self.seq_length]      ,      vocab_size=2     )

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_mask,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										    encoder_hidden_states,
										    encoder_attention_mask,
										)


				def 		__magic_name__(	self      :Optional[int]      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :List[str]      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :Union[str, Any]      ,      lowerCAmelCase__      :List[Any]     )		->      Optional[int]:
										__SCREAMING_SNAKE_CASE						:					str		      =		RemBertModel(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		model(lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__     )
										__SCREAMING_SNAKE_CASE						:					int		      =		model(lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__     )
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		model(lowerCAmelCase__     )
										self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)     )


				def 		__magic_name__(	self      :Optional[int]      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :List[str]      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :int      ,      lowerCAmelCase__      :Any      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :Any      ,      )		->      Any:
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		True
										__SCREAMING_SNAKE_CASE						:					Tuple		      =		RemBertModel(lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      encoder_hidden_states=lowerCAmelCase__      ,      encoder_attention_mask=lowerCAmelCase__      ,      )
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      encoder_hidden_states=lowerCAmelCase__      ,      )
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		model(lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__     )
										self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)     )


				def 		__magic_name__(	self      :Optional[Any]      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Any      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :str      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :int      ,      )		->      Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:					Dict		      =		RemBertForCausalLM(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					Any		      =		model(lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      labels=lowerCAmelCase__     )
										self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)     )


				def 		__magic_name__(	self      :Optional[int]      ,      lowerCAmelCase__      :int      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :Any      ,      lowerCAmelCase__      :List[str]     )		->      Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:					int		      =		RemBertForMaskedLM(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					Dict		      =		model(lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      labels=lowerCAmelCase__     )
										self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)     )


				def 		__magic_name__(	self      :Tuple      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :Any      ,      lowerCAmelCase__      :int      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :str      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :int      ,      lowerCAmelCase__      :str      ,      )		->      Dict:
										__SCREAMING_SNAKE_CASE						:					Dict		      =		True
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		True
										__SCREAMING_SNAKE_CASE						:					str		      =		RemBertForCausalLM(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()

										# first forward pass
										__SCREAMING_SNAKE_CASE						:					Dict		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      encoder_hidden_states=lowerCAmelCase__      ,      encoder_attention_mask=lowerCAmelCase__      ,      use_cache=lowerCAmelCase__      ,      )
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		ids_tensor((self.batch_size, 3)      ,      config.vocab_size     )
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		ids_tensor((self.batch_size, 3)      ,      vocab_size=2     )

										# append to next input_ids and
										__SCREAMING_SNAKE_CASE						:					str		      =		torch.cat([input_ids, next_tokens]      ,      dim=-1     )
										__SCREAMING_SNAKE_CASE						:					str		      =		torch.cat([input_mask, next_mask]      ,      dim=-1     )

										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      encoder_hidden_states=lowerCAmelCase__      ,      encoder_attention_mask=lowerCAmelCase__      ,      output_hidden_states=lowerCAmelCase__      ,      )['''hidden_states'''][0]
										__SCREAMING_SNAKE_CASE						:					int		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      encoder_hidden_states=lowerCAmelCase__      ,      encoder_attention_mask=lowerCAmelCase__      ,      past_key_values=lowerCAmelCase__      ,      output_hidden_states=lowerCAmelCase__      ,      )['''hidden_states'''][0]

										# select random slice
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		ids_tensor((1,)      ,      output_from_past.shape[-1]     ).item()
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		output_from_no_past[:, -3:, random_slice_idx].detach()
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(lowerCAmelCase__      ,      lowerCAmelCase__      ,      atol=1E-3     )     )


				def 		__magic_name__(	self      :Any      ,      lowerCAmelCase__      :Union[str, Any]      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Any      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :Union[str, Any]     )		->      Any:
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		RemBertForQuestionAnswering(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      start_positions=lowerCAmelCase__      ,      end_positions=lowerCAmelCase__      ,      )
										self.parent.assertEqual(result.start_logits.shape      ,      (self.batch_size, self.seq_length)     )
										self.parent.assertEqual(result.end_logits.shape      ,      (self.batch_size, self.seq_length)     )


				def 		__magic_name__(	self      :Optional[Any]      ,      lowerCAmelCase__      :Union[str, Any]      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Union[str, Any]      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :Optional[Any]     )		->      Tuple:
										__SCREAMING_SNAKE_CASE						:					Dict		      =		self.num_labels
										__SCREAMING_SNAKE_CASE						:					Any		      =		RemBertForSequenceClassification(lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					Dict		      =		model(lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      labels=lowerCAmelCase__     )
										self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_labels)     )


				def 		__magic_name__(	self      :int      ,      lowerCAmelCase__      :str      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :Union[str, Any]      ,      lowerCAmelCase__      :Any      ,      lowerCAmelCase__      :Optional[Any]      ,      lowerCAmelCase__      :List[str]     )		->      Optional[Any]:
										__SCREAMING_SNAKE_CASE						:					Any		      =		self.num_labels
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		RemBertForTokenClassification(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					Optional[Any]		      =		model(lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      labels=lowerCAmelCase__     )
										self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.num_labels)     )


				def 		__magic_name__(	self      :List[str]      ,      lowerCAmelCase__      :Dict      ,      lowerCAmelCase__      :Tuple      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :List[Any]      ,      lowerCAmelCase__      :Union[str, Any]      ,      lowerCAmelCase__      :Optional[int]      ,      lowerCAmelCase__      :Union[str, Any]     )		->      List[str]:
										__SCREAMING_SNAKE_CASE						:					Any		      =		self.num_choices
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		RemBertForMultipleChoice(config=lowerCAmelCase__     )
										model.to(lowerCAmelCase__     )
										model.eval()
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		input_ids.unsqueeze(1     ).expand(-1      ,      self.num_choices      ,      -1     ).contiguous()
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		token_type_ids.unsqueeze(1     ).expand(-1      ,      self.num_choices      ,      -1     ).contiguous()
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		input_mask.unsqueeze(1     ).expand(-1      ,      self.num_choices      ,      -1     ).contiguous()
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		model(
										    lowerCAmelCase__      ,      attention_mask=lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      labels=lowerCAmelCase__      ,      )
										self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_choices)     )



				def 		__magic_name__(	self      :Union[str, Any]     )		->      Any:
										__SCREAMING_SNAKE_CASE						:					int		      =		self.prepare_config_and_inputs()
										(
										    (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      
										)						:					int		      =		config_and_inputs
										__SCREAMING_SNAKE_CASE						:					Any		      =		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict




@require_torch
class 	_lowercase				(      A__						,      A__						,      unittest.TestCase	):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__    :      Dict      =		(
				    (
				        RemBertModel,
				        RemBertForMaskedLM,
				        RemBertForCausalLM,
				        RemBertForMultipleChoice,
				        RemBertForQuestionAnswering,
				        RemBertForSequenceClassification,
				        RemBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				SCREAMING_SNAKE_CASE__    :      Tuple      =		(RemBertForCausalLM,) if is_torch_available() else ()
				SCREAMING_SNAKE_CASE__    :      Tuple      =		(
				    {
				        '''feature-extraction''': RemBertModel,
				        '''fill-mask''': RemBertForMaskedLM,
				        '''question-answering''': RemBertForQuestionAnswering,
				        '''text-classification''': RemBertForSequenceClassification,
				        '''text-generation''': RemBertForCausalLM,
				        '''token-classification''': RemBertForTokenClassification,
				        '''zero-shot''': RemBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)


				def 		__magic_name__(	self      :List[str]     )		->      List[str]:
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		RemBertModelTester(self     )
										__SCREAMING_SNAKE_CASE						:					Dict		      =		ConfigTester(self      ,      config_class=lowerCAmelCase__      ,      hidden_size=37     )


				def 		__magic_name__(	self      :Dict     )		->      int:
										self.config_tester.run_common_tests()


				def 		__magic_name__(	self      :Any     )		->      List[str]:
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Optional[int]     )		->      Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:					int		      =		self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		type
																self.model_tester.create_and_check_model(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Optional[int]     )		->      Dict:
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :List[Any]     )		->      int:
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Dict     )		->      Dict:
										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Optional[int]     )		->      Optional[Any]:
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Any     )		->      Dict:
										__SCREAMING_SNAKE_CASE						:					List[str]		      =		self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Optional[int]     )		->      List[str]:
										__SCREAMING_SNAKE_CASE						:					str		      =		self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :str     )		->      Dict:
										__SCREAMING_SNAKE_CASE						:					int		      =		self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase__     )


				def 		__magic_name__(	self      :Optional[int]     )		->      Optional[int]:
										# This regression test was failing with PyTorch < 1.3
										(
										    (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      (
										    __SCREAMING_SNAKE_CASE
										)      ,      
										)						:					Optional[Any]		      =		self.model_tester.prepare_config_and_inputs_for_decoder()

										__SCREAMING_SNAKE_CASE						:					Any		      =		None

										self.model_tester.create_and_check_model_as_decoder(
										    lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ,      )



				@slow
				def 		__magic_name__(	self      :Any     )		->      Union[str, Any]:
										for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__SCREAMING_SNAKE_CASE						:					List[Any]		      =		RemBertModel.from_pretrained(lowerCAmelCase__     )
																self.assertIsNotNone(lowerCAmelCase__     )




@require_torch
class 	_lowercase				(      unittest.TestCase	):






				'''simple docstring'''



				@slow
				def 		__magic_name__(	self      :str     )		->      List[Any]:
										# Test exact values at the last hidden layer
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		RemBertModel.from_pretrained('''google/rembert'''     )
										__SCREAMING_SNAKE_CASE						:					List[Any]		      =		torch.tensor([[312, 56_498, 313, 2_125, 313]]     )
										__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		torch.tensor([[0, 0, 0, 1, 1]]     )
										with torch.no_grad():
																__SCREAMING_SNAKE_CASE						:					Union[str, Any]		      =		model(lowerCAmelCase__      ,      token_type_ids=lowerCAmelCase__      ,      output_hidden_states=lowerCAmelCase__     )

										__SCREAMING_SNAKE_CASE						:					List[str]		      =		1_152

										__SCREAMING_SNAKE_CASE						:					Optional[int]		      =		torch.Size((1, 5, hidden_size)     )
										self.assertEqual(output['''last_hidden_state'''].shape      ,      lowerCAmelCase__     )

										__SCREAMING_SNAKE_CASE						:					str		      =		torch.tensor(
										    [
										        [
										            [0.0754, -0.2022, 0.1904],
										            [-0.3354, -0.3692, -0.4791],
										            [-0.2314, -0.6729, -0.0749],
										            [-0.0396, -0.3105, -0.4234],
										            [-0.1571, -0.0525, 0.5353],
										        ]
										    ]     )

										# Running on the original tf implementation gives slightly different results here.
										# Not clear why this variations is present
										# TODO: Find reason for discrepancy
										# expected_original_implementation = [[
										#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
										#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
										#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
										#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
										#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
										# ]]

										self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]      ,      lowerCAmelCase__      ,      atol=1E-4     )     )
