






from typing import Callable, List, Optional, Union

import torch

from ...models import UNetaDModel
from ...schedulers import CMStochasticIterativeScheduler
from ...utils import (
    is_accelerate_available,
    is_accelerate_version,
    logging,
    randn_tensor,
    replace_example_docstring,
)
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput


A__     :      str             =			logging.get_logger(__name__)  # pylint: disable=invalid-name


A__     :      List[Any]             =			'\n    Examples:\n        ```py\n        >>> import torch\n\n        >>> from diffusers import ConsistencyModelPipeline\n\n        >>> device = "cuda"\n        >>> # Load the cd_imagenet64_l2 checkpoint.\n        >>> model_id_or_path = "openai/diffusers-cd_imagenet64_l2"\n        >>> pipe = ConsistencyModelPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16)\n        >>> pipe.to(device)\n\n        >>> # Onestep Sampling\n        >>> image = pipe(num_inference_steps=1).images[0]\n        >>> image.save("cd_imagenet64_l2_onestep_sample.png")\n\n        >>> # Onestep sampling, class-conditional image generation\n        >>> # ImageNet-64 class label 145 corresponds to king penguins\n        >>> image = pipe(num_inference_steps=1, class_labels=145).images[0]\n        >>> image.save("cd_imagenet64_l2_onestep_sample_penguin.png")\n\n        >>> # Multistep sampling, class-conditional image generation\n        >>> # Timesteps can be explicitly specified; the particular timesteps below are from the original Github repo:\n        >>> # https://github.com/openai/consistency_models/blob/main/scripts/launch.sh#L77\n        >>> image = pipe(num_inference_steps=None, timesteps=[22, 0], class_labels=145).images[0]\n        >>> image.save("cd_imagenet64_l2_multistep_sample_penguin.png")\n        ```\n'
class 				_UpperCAmelCase (	A__     ):

				"""simple docstring"""




				def __init__(					self						:   List[str],  lowerCamelCase						:   UNetaDModel,  lowerCamelCase						:   CMStochasticIterativeScheduler  ):

								'''simple docstring'''
								super().__init__()

								self.register_modules(
								    unet=lowerCamelCase,  scheduler=lowerCamelCase,  )

								lowercase__       			=      None




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   str=0  ):

								'''simple docstring'''
								if is_accelerate_available() and is_accelerate_version('''>=''',  '''0.14.0'''  ):
												from accelerate import cpu_offload
								else:
												raise ImportError('''`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher'''  )

								lowercase__       			=      torch.device(F"""cuda:{gpu_id}"""  )

								if self.device.type != "cpu":
												self.to('''cpu''',  silence_dtype_warnings=lowerCamelCase  )
												torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

								for cpu_offloaded_model in [self.unet]:
												cpu_offload(lowerCamelCase,  lowerCamelCase  )

								if self.safety_checker is not None:
												cpu_offload(self.safety_checker,  execution_device=lowerCamelCase,  offload_buffers=lowerCamelCase  )




				def     lowercase__       (					self						:   Optional[int],  lowerCamelCase						:   Optional[Any]=0  ):

								'''simple docstring'''
								if is_accelerate_available() and is_accelerate_version('''>=''',  '''0.17.0.dev0'''  ):
												from accelerate import cpu_offload_with_hook
								else:
												raise ImportError('''`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.'''  )

								lowercase__       			=      torch.device(F"""cuda:{gpu_id}"""  )

								if self.device.type != "cpu":
												self.to('''cpu''',  silence_dtype_warnings=lowerCamelCase  )
												torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

								lowercase__       			=      None
								for cpu_offloaded_model in [self.unet]:
												lowercase__			,						lowercase__       			=      cpu_offload_with_hook(lowerCamelCase,  lowerCamelCase,  prev_module_hook=lowerCamelCase  )

								if self.safety_checker is not None:
												lowercase__			,						lowercase__       			=      cpu_offload_with_hook(self.safety_checker,  lowerCamelCase,  prev_module_hook=lowerCamelCase  )

								# We'll offload the last model manually.
								lowercase__       			=      hook




				@property
				# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
				def     lowercase__       (					self						:   List[Any]  ):

								'''simple docstring'''
								if not hasattr(self.unet,  '''_hf_hook'''  ):
												return self.device
								for module in self.unet.modules():
												if (
												    hasattr(lowerCamelCase,  '''_hf_hook'''  )
												    and hasattr(module._hf_hook,  '''execution_device'''  )
												    and module._hf_hook.execution_device is not None
												):
																return torch.device(module._hf_hook.execution_device  )
								return self.device




				def     lowercase__       (					self						:   Any,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   List[str],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Any,  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   str,  lowerCamelCase						:   Union[str, Any]=None  ):

								'''simple docstring'''
								lowercase__       			=      (batch_size, num_channels, height, width)
								if isinstance(lowerCamelCase,  lowerCamelCase  ) and len(lowerCamelCase  ) != batch_size:
												raise ValueError(
												    F"""You have passed a list of generators of length {len(lowerCamelCase  )}, but requested an effective batch"""
												    F""" size of {batch_size}. Make sure the batch size matches the length of the generators."""  )

								if latents is None:
												lowercase__       			=      randn_tensor(lowerCamelCase,  generator=lowerCamelCase,  device=lowerCamelCase,  dtype=lowerCamelCase  )
								else:
												lowercase__       			=      latents.to(device=lowerCamelCase,  dtype=lowerCamelCase  )

								# scale the initial noise by the standard deviation required by the scheduler
								lowercase__       			=      latents * self.scheduler.init_noise_sigma
								return latents




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   torch.FloatTensor,  lowerCamelCase						:   str = "pil"  ):

								'''simple docstring'''
								if output_type not in ["pt", "np", "pil"]:
												raise ValueError(
												    F"""output_type={output_type} is not supported. Make sure to choose one of ['pt', 'np', or 'pil']"""  )

								# Equivalent to diffusers.VaeImageProcessor.denormalize
								lowercase__       			=      (sample / 2 + 0.5).clamp(0,  1  )
								if output_type == "pt":
												return sample

								# Equivalent to diffusers.VaeImageProcessor.pt_to_numpy
								lowercase__       			=      sample.cpu().permute(0,  2,  3,  1  ).numpy()
								if output_type == "np":
												return sample

								# Output_type must be 'pil'
								lowercase__       			=      self.numpy_to_pil(lowerCamelCase  )
								return sample




				def     lowercase__       (					self						:   Union[str, Any],  lowerCamelCase						:   Any,  lowerCamelCase						:   str,  lowerCamelCase						:   Optional[Any]=None  ):

								'''simple docstring'''
								if self.unet.config.num_class_embeds is not None:
												if isinstance(lowerCamelCase,  lowerCamelCase  ):
																lowercase__       			=      torch.tensor(lowerCamelCase,  dtype=torch.int  )
												elif isinstance(lowerCamelCase,  lowerCamelCase  ):
																assert batch_size == 1, "Batch size must be 1 if classes is an int"
																lowercase__       			=      torch.tensor([class_labels],  dtype=torch.int  )
												elif class_labels is None:
																# Randomly generate batch_size class labels
																# TODO: should use generator here? int analogue of randn_tensor is not exposed in ...utils
																lowercase__       			=      torch.randint(0,  self.unet.config.num_class_embeds,  size=(batch_size,)  )
												lowercase__       			=      class_labels.to(lowerCamelCase  )
								else:
												lowercase__       			=      None
								return class_labels




				def     lowercase__       (					self						:   int,  lowerCamelCase						:   List[str],  lowerCamelCase						:   str,  lowerCamelCase						:   Any,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   int,  lowerCamelCase						:   Optional[int]  ):

								'''simple docstring'''
								if num_inference_steps is None and timesteps is None:
												raise ValueError('''Exactly one of `num_inference_steps` or `timesteps` must be supplied.'''  )

								if num_inference_steps is not None and timesteps is not None:
												logger.warning(
												    F"""Both `num_inference_steps`: {num_inference_steps} and `timesteps`: {timesteps} are supplied;"""
												    ''' `timesteps` will be used over `num_inference_steps`.'''  )

								if latents is not None:
												lowercase__       			=      (batch_size, 3, img_size, img_size)
												if latents.shape != expected_shape:
																raise ValueError(F"""The shape of latents is {latents.shape} but is expected to be {expected_shape}."""  )

								if (callback_steps is None) or (
								    callback_steps is not None and (not isinstance(lowerCamelCase,  lowerCamelCase  ) or callback_steps <= 0)
								):
												raise ValueError(
												    F"""`callback_steps` has to be a positive integer but is {callback_steps} of type"""
												    F""" {type(lowerCamelCase  )}."""  )




				@torch.no_grad()
				@replace_example_docstring(lowerCamelCase  )
				def __call__(					self						:   Union[str, Any],  lowerCamelCase						:   int = 1,  lowerCamelCase						:   Optional[Union[torch.Tensor, List[int], int]] = None,  lowerCamelCase						:   int = 1,  lowerCamelCase						:   List[int] = None,  lowerCamelCase						:   Optional[Union[torch.Generator, List[torch.Generator]]] = None,  lowerCamelCase						:   Optional[torch.FloatTensor] = None,  lowerCamelCase						:   Optional[str] = "pil",  lowerCamelCase						:   bool = True,  lowerCamelCase						:   Optional[Callable[[int, int, torch.FloatTensor], None]] = None,  lowerCamelCase						:   int = 1,  ):

								'''simple docstring'''
								lowercase__       			=      self.unet.config.sample_size
								lowercase__       			=      self._execution_device

								# 1. Check inputs
								self.check_inputs(lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase  )

								# 2. Prepare image latents
								# Sample image latents x_0 ~ N(0, sigma_0^2 * I)
								lowercase__       			=      self.prepare_latents(
								    batch_size=lowerCamelCase,  num_channels=self.unet.config.in_channels,  height=lowerCamelCase,  width=lowerCamelCase,  dtype=self.unet.dtype,  device=lowerCamelCase,  generator=lowerCamelCase,  latents=lowerCamelCase,  )

								# 3. Handle class_labels for class-conditional models
								lowercase__       			=      self.prepare_class_labels(lowerCamelCase,  lowerCamelCase,  class_labels=lowerCamelCase  )

								# 4. Prepare timesteps
								if timesteps is not None:
												self.scheduler.set_timesteps(timesteps=lowerCamelCase,  device=lowerCamelCase  )
												lowercase__       			=      self.scheduler.timesteps
												lowercase__       			=      len(lowerCamelCase  )
								else:
												self.scheduler.set_timesteps(lowerCamelCase  )
												lowercase__       			=      self.scheduler.timesteps

								# 5. Denoising loop
								# Multistep sampling: implements Algorithm 1 in the paper
								with self.progress_bar(total=lowerCamelCase  ) as progress_bar:
												for i, t in enumerate(lowerCamelCase  ):
																lowercase__       			=      self.scheduler.scale_model_input(lowerCamelCase,  lowerCamelCase  )
																lowercase__       			=      self.unet(lowerCamelCase,  lowerCamelCase,  class_labels=lowerCamelCase,  return_dict=lowerCamelCase  )[0]

																lowercase__       			=      self.scheduler.step(lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  generator=lowerCamelCase  )[0]

																# call the callback, if provided
																progress_bar.update()
																if callback is not None and i % callback_steps == 0:
																				callback(lowerCamelCase,  lowerCamelCase,  lowerCamelCase  )

        # 6. Post-process image sample
								lowercase__       			=      self.postprocess_image(lowerCamelCase,  output_type=lowerCamelCase  )

								# Offload last model to CPU
								if hasattr(self,  '''final_offload_hook'''  ) and self.final_offload_hook is not None:
												self.final_offload_hook.offload()

								if not return_dict:
												return (image,)

								return ImagePipelineOutput(images=lowerCamelCase  )












import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RemBertConfig,
							    RemBertForCausalLM,
							    RemBertForMaskedLM,
							    RemBertForMultipleChoice,
							    RemBertForQuestionAnswering,
							    RemBertForSequenceClassification,
							    RemBertForTokenClassification,
							    RemBertModel,
							)
							from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class 				_UpperCAmelCase :

				"""simple docstring"""




				def __init__(					self						:   Optional[int],  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   int=13,  lowerCamelCase						:   str=7,  lowerCamelCase						:   int=True,  lowerCamelCase						:   Optional[Any]=True,  lowerCamelCase						:   List[str]=True,  lowerCamelCase						:   str=True,  lowerCamelCase						:   List[str]=99,  lowerCamelCase						:   Any=32,  lowerCamelCase						:   int=18,  lowerCamelCase						:   Optional[int]=43,  lowerCamelCase						:   Optional[Any]=5,  lowerCamelCase						:   int=4,  lowerCamelCase						:   Union[str, Any]=37,  lowerCamelCase						:   List[str]="gelu",  lowerCamelCase						:   List[str]=0.1,  lowerCamelCase						:   Dict=0.1,  lowerCamelCase						:   Dict=512,  lowerCamelCase						:   str=16,  lowerCamelCase						:   Optional[Any]=2,  lowerCamelCase						:   int=0.02,  lowerCamelCase						:   Optional[int]=3,  lowerCamelCase						:   Optional[int]=4,  lowerCamelCase						:   Union[str, Any]=None,  ):

								'''simple docstring'''
								lowercase__       			=      parent
								lowercase__       			=      batch_size
								lowercase__       			=      seq_length
								lowercase__       			=      is_training
								lowercase__       			=      use_input_mask
								lowercase__       			=      use_token_type_ids
								lowercase__       			=      use_labels
								lowercase__       			=      vocab_size
								lowercase__       			=      hidden_size
								lowercase__       			=      input_embedding_size
								lowercase__       			=      output_embedding_size
								lowercase__       			=      num_hidden_layers
								lowercase__       			=      num_attention_heads
								lowercase__       			=      intermediate_size
								lowercase__       			=      hidden_act
								lowercase__       			=      hidden_dropout_prob
								lowercase__       			=      attention_probs_dropout_prob
								lowercase__       			=      max_position_embeddings
								lowercase__       			=      type_vocab_size
								lowercase__       			=      type_sequence_label_size
								lowercase__       			=      initializer_range
								lowercase__       			=      num_labels
								lowercase__       			=      num_choices
								lowercase__       			=      scope




				def     lowercase__       (					self						:   Optional[Any]  ):

								'''simple docstring'''
								lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  self.vocab_size  )

								lowercase__       			=      None
								if self.use_input_mask:
												lowercase__       			=      random_attention_mask([self.batch_size, self.seq_length]  )

								lowercase__       			=      None
								if self.use_token_type_ids:
												lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  self.type_vocab_size  )

								lowercase__       			=      None
								lowercase__       			=      None
								lowercase__       			=      None
								if self.use_labels:
												lowercase__       			=      ids_tensor([self.batch_size],  self.type_sequence_label_size  )
												lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  self.num_labels  )
												lowercase__       			=      ids_tensor([self.batch_size],  self.num_choices  )

								lowercase__       			=      RemBertConfig(
								    vocab_size=self.vocab_size,  hidden_size=self.hidden_size,  input_embedding_size=self.input_embedding_size,  output_embedding_size=self.output_embedding_size,  num_hidden_layers=self.num_hidden_layers,  num_attention_heads=self.num_attention_heads,  intermediate_size=self.intermediate_size,  hidden_act=self.hidden_act,  hidden_dropout_prob=self.hidden_dropout_prob,  attention_probs_dropout_prob=self.attention_probs_dropout_prob,  max_position_embeddings=self.max_position_embeddings,  type_vocab_size=self.type_vocab_size,  is_decoder=lowerCamelCase,  initializer_range=self.initializer_range,  )

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def     lowercase__       (					self						:   Union[str, Any]  ):

								'''simple docstring'''
								(
								    (
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						
								)       			=      self.prepare_config_and_inputs()

								lowercase__       			=      True
								lowercase__       			=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
								lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  vocab_size=2  )

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								    encoder_hidden_states,
								    encoder_attention_mask,
								)




				def     lowercase__       (					self						:   Tuple,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   List[str],  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   int,  lowerCamelCase						:   List[Any],  lowerCamelCase						:   str  ):

								'''simple docstring'''
								lowercase__       			=      RemBertModel(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase  )
								lowercase__       			=      model(lowerCamelCase,  token_type_ids=lowerCamelCase  )
								lowercase__       			=      model(lowerCamelCase  )
								self.parent.assertEqual(result.last_hidden_state.shape,  (self.batch_size, self.seq_length, self.hidden_size)  )




				def     lowercase__       (					self						:   Optional[Any],  lowerCamelCase						:   List[str],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Dict,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   str,  lowerCamelCase						:   Any,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Any,  ):

								'''simple docstring'''
								lowercase__       			=      True
								lowercase__       			=      RemBertModel(lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  )
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  )
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase  )
								self.parent.assertEqual(result.last_hidden_state.shape,  (self.batch_size, self.seq_length, self.hidden_size)  )




				def     lowercase__       (					self						:   str,  lowerCamelCase						:   str,  lowerCamelCase						:   str,  lowerCamelCase						:   int,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Optional[Any],  ):

								'''simple docstring'''
								lowercase__       			=      RemBertForCausalLM(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.seq_length, self.vocab_size)  )




				def     lowercase__       (					self						:   Any,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Dict,  lowerCamelCase						:   str,  lowerCamelCase						:   List[str],  lowerCamelCase						:   Any,  lowerCamelCase						:   List[str]  ):

								'''simple docstring'''
								lowercase__       			=      RemBertForMaskedLM(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.seq_length, self.vocab_size)  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   List[str],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Any,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   int,  lowerCamelCase						:   Dict,  ):

								'''simple docstring'''
								lowercase__       			=      True
								lowercase__       			=      True
								lowercase__       			=      RemBertForCausalLM(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()

								# first forward pass
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  use_cache=lowerCamelCase,  )
								lowercase__       			=      outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								lowercase__       			=      ids_tensor((self.batch_size, 3),  config.vocab_size  )
								lowercase__       			=      ids_tensor((self.batch_size, 3),  vocab_size=2  )

								# append to next input_ids and
								lowercase__       			=      torch.cat([input_ids, next_tokens],  dim=-1  )
								lowercase__       			=      torch.cat([input_mask, next_mask],  dim=-1  )

								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  output_hidden_states=lowerCamelCase,  )['''hidden_states'''][0]
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  past_key_values=lowerCamelCase,  output_hidden_states=lowerCamelCase,  )['''hidden_states'''][0]

								# select random slice
								lowercase__       			=      ids_tensor((1,),  output_from_past.shape[-1]  ).item()
								lowercase__       			=      output_from_no_past[:, -3:, random_slice_idx].detach()
								lowercase__       			=      output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(lowerCamelCase,  lowerCamelCase,  atol=1E-3  )  )




				def     lowercase__       (					self						:   Optional[int],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   List[str],  lowerCamelCase						:   List[Any]  ):

								'''simple docstring'''
								lowercase__       			=      RemBertForQuestionAnswering(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  start_positions=lowerCamelCase,  end_positions=lowerCamelCase,  )
								self.parent.assertEqual(result.start_logits.shape,  (self.batch_size, self.seq_length)  )
								self.parent.assertEqual(result.end_logits.shape,  (self.batch_size, self.seq_length)  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Union[str, Any]  ):

								'''simple docstring'''
								lowercase__       			=      self.num_labels
								lowercase__       			=      RemBertForSequenceClassification(lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.num_labels)  )




				def     lowercase__       (					self						:   Tuple,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Dict,  lowerCamelCase						:   str,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Optional[int]  ):

								'''simple docstring'''
								lowercase__       			=      self.num_labels
								lowercase__       			=      RemBertForTokenClassification(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.seq_length, self.num_labels)  )




				def     lowercase__       (					self						:   Optional[int],  lowerCamelCase						:   Any,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   int,  lowerCamelCase						:   str,  lowerCamelCase						:   int,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   Any  ):

								'''simple docstring'''
								lowercase__       			=      self.num_choices
								lowercase__       			=      RemBertForMultipleChoice(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      input_ids.unsqueeze(1  ).expand(-1,  self.num_choices,  -1  ).contiguous()
								lowercase__       			=      token_type_ids.unsqueeze(1  ).expand(-1,  self.num_choices,  -1  ).contiguous()
								lowercase__       			=      input_mask.unsqueeze(1  ).expand(-1,  self.num_choices,  -1  ).contiguous()
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase,  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.num_choices)  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								lowercase__       			=      self.prepare_config_and_inputs()
								(
								    (
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						
								)       			=      config_and_inputs
								lowercase__       			=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
								return config, inputs_dict



@require_torch
class 				_UpperCAmelCase (	A__		,A__		,unittest.TestCase     ):

				"""simple docstring"""




				lowercase__    							=	(
				    (
				        RemBertModel,
				        RemBertForMaskedLM,
				        RemBertForCausalLM,
				        RemBertForMultipleChoice,
				        RemBertForQuestionAnswering,
				        RemBertForSequenceClassification,
				        RemBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				lowercase__    							=	(RemBertForCausalLM,) if is_torch_available() else ()
				lowercase__    							=	(
				    {
				        """feature-extraction""": RemBertModel,
				        """fill-mask""": RemBertForMaskedLM,
				        """question-answering""": RemBertForQuestionAnswering,
				        """text-classification""": RemBertForSequenceClassification,
				        """text-generation""": RemBertForCausalLM,
				        """token-classification""": RemBertForTokenClassification,
				        """zero-shot""": RemBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)




				def     lowercase__       (					self						:   str  ):

								'''simple docstring'''
								lowercase__       			=      RemBertModelTester(self  )
								lowercase__       			=      ConfigTester(self,  config_class=lowerCamelCase,  hidden_size=37  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								self.config_tester.run_common_tests()




				def     lowercase__       (					self						:   List[Any]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCamelCase  )




				def     lowercase__       (					self						:   Dict  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
												lowercase__       			=      type
												self.model_tester.create_and_check_model(*lowerCamelCase  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase  )




				def     lowercase__       (					self						:   Optional[int]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase  )




				def     lowercase__       (					self						:   Dict  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCamelCase  )




				def     lowercase__       (					self						:   Tuple  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*lowerCamelCase  )




				def     lowercase__       (					self						:   Union[str, Any]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase  )




				def     lowercase__       (					self						:   Tuple  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*lowerCamelCase  )




				def     lowercase__       (					self						:   Any  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								# This regression test was failing with PyTorch < 1.3
								(
								    (
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						
								)       			=      self.model_tester.prepare_config_and_inputs_for_decoder()

								lowercase__       			=      None

								self.model_tester.create_and_check_model_as_decoder(
								    lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  )




				@slow
				def     lowercase__       (					self						:   int  ):

								'''simple docstring'''
								for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												lowercase__       			=      RemBertModel.from_pretrained(lowerCamelCase  )
												self.assertIsNotNone(lowerCamelCase  )



@require_torch
class 				_UpperCAmelCase (	unittest.TestCase     ):

				"""simple docstring"""




				@slow
				def     lowercase__       (					self						:   int  ):

								'''simple docstring'''
								# Test exact values at the last hidden layer
								lowercase__       			=      RemBertModel.from_pretrained('''google/rembert'''  )
								lowercase__       			=      torch.tensor([[312, 56_498, 313, 2_125, 313]]  )
								lowercase__       			=      torch.tensor([[0, 0, 0, 1, 1]]  )
								with torch.no_grad():
												lowercase__       			=      model(lowerCamelCase,  token_type_ids=lowerCamelCase,  output_hidden_states=lowerCamelCase  )

								lowercase__       			=      1_152

								lowercase__       			=      torch.Size((1, 5, hidden_size)  )
								self.assertEqual(output['''last_hidden_state'''].shape,  lowerCamelCase  )

								lowercase__       			=      torch.tensor(
								    [
								        [
								            [0.0754, -0.2022, 0.1904],
								            [-0.3354, -0.3692, -0.4791],
								            [-0.2314, -0.6729, -0.0749],
								            [-0.0396, -0.3105, -0.4234],
								            [-0.1571, -0.0525, 0.5353],
								        ]
								    ]  )

								# Running on the original tf implementation gives slightly different results here.
								# Not clear why this variations is present
								# TODO: Find reason for discrepancy
								# expected_original_implementation = [[
								#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
								#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
								#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
								#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
								#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
								# ]]

								self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3],  lowerCamelCase,  atol=1E-4  )  )




