
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     RemBertConfig,
     RemBertForCausalLM,
     RemBertForMaskedLM,
     RemBertForMultipleChoice,
     RemBertForQuestionAnswering,
     RemBertForSequenceClassification,
     RemBertForTokenClassification,
     RemBertModel,
 )
 from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class       UpperCAmelCase_						:



      """simple docstring"""
      def __init__(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_=13   ,    SCREAMING_SNAKE_CASE_=7   ,    SCREAMING_SNAKE_CASE_=True   ,    SCREAMING_SNAKE_CASE_=True   ,    SCREAMING_SNAKE_CASE_=True   ,    SCREAMING_SNAKE_CASE_=True   ,    SCREAMING_SNAKE_CASE_=99   ,    SCREAMING_SNAKE_CASE_=32   ,    SCREAMING_SNAKE_CASE_=18   ,    SCREAMING_SNAKE_CASE_=43   ,    SCREAMING_SNAKE_CASE_=5   ,    SCREAMING_SNAKE_CASE_=4   ,    SCREAMING_SNAKE_CASE_=37   ,    SCREAMING_SNAKE_CASE_="gelu"   ,    SCREAMING_SNAKE_CASE_=0.1   ,    SCREAMING_SNAKE_CASE_=0.1   ,    SCREAMING_SNAKE_CASE_=512   ,    SCREAMING_SNAKE_CASE_=16   ,    SCREAMING_SNAKE_CASE_=2   ,    SCREAMING_SNAKE_CASE_=0.02   ,    SCREAMING_SNAKE_CASE_=3   ,    SCREAMING_SNAKE_CASE_=4   ,    SCREAMING_SNAKE_CASE_=None   ,    )    ->				Optional[int]:
          UpperCamelCase	:Optional[int]     							=       parent
          UpperCamelCase	:Optional[Any]     							=       batch_size
          UpperCamelCase	:str     							=       seq_length
          UpperCamelCase	:int     							=       is_training
          UpperCamelCase	:Tuple     							=       use_input_mask
          UpperCamelCase	:Dict     							=       use_token_type_ids
          UpperCamelCase	:Dict     							=       use_labels
          UpperCamelCase	:Dict     							=       vocab_size
          UpperCamelCase	:Union[str, Any]     							=       hidden_size
          UpperCamelCase	:Dict     							=       input_embedding_size
          UpperCamelCase	:str     							=       output_embedding_size
          UpperCamelCase	:Dict     							=       num_hidden_layers
          UpperCamelCase	:List[Any]     							=       num_attention_heads
          UpperCamelCase	:int     							=       intermediate_size
          UpperCamelCase	:Tuple     							=       hidden_act
          UpperCamelCase	:Any     							=       hidden_dropout_prob
          UpperCamelCase	:Tuple     							=       attention_probs_dropout_prob
          UpperCamelCase	:Optional[int]     							=       max_position_embeddings
          UpperCamelCase	:str     							=       type_vocab_size
          UpperCamelCase	:Union[str, Any]     							=       type_sequence_label_size
          UpperCamelCase	:Optional[int]     							=       initializer_range
          UpperCamelCase	:int     							=       num_labels
          UpperCamelCase	:Tuple     							=       num_choices
          UpperCamelCase	:int     							=       scope
      def    UpperCAmelCase						(    self			)    ->				str:
          UpperCamelCase	:Union[str, Any]     							=       ids_tensor([self.batch_size, self.seq_length]   ,    self.vocab_size			)

          UpperCamelCase	:Optional[int]     							=       None
          if self.use_input_mask:
              UpperCamelCase	:Dict     							=       random_attention_mask([self.batch_size, self.seq_length]			)

          UpperCamelCase	:Tuple     							=       None
          if self.use_token_type_ids:
              UpperCamelCase	:Optional[Any]     							=       ids_tensor([self.batch_size, self.seq_length]   ,    self.type_vocab_size			)

          UpperCamelCase	:Optional[int]     							=       None
          UpperCamelCase	:Union[str, Any]     							=       None
          UpperCamelCase	:Optional[int]     							=       None
          if self.use_labels:
              UpperCamelCase	:int     							=       ids_tensor([self.batch_size]   ,    self.type_sequence_label_size			)
              UpperCamelCase	:str     							=       ids_tensor([self.batch_size, self.seq_length]   ,    self.num_labels			)
              UpperCamelCase	:List[Any]     							=       ids_tensor([self.batch_size]   ,    self.num_choices			)

          UpperCamelCase	:List[Any]     							=       RemBertConfig(
              vocab_size=self.vocab_size   ,    hidden_size=self.hidden_size   ,    input_embedding_size=self.input_embedding_size   ,    output_embedding_size=self.output_embedding_size   ,    num_hidden_layers=self.num_hidden_layers   ,    num_attention_heads=self.num_attention_heads   ,    intermediate_size=self.intermediate_size   ,    hidden_act=self.hidden_act   ,    hidden_dropout_prob=self.hidden_dropout_prob   ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,    max_position_embeddings=self.max_position_embeddings   ,    type_vocab_size=self.type_vocab_size   ,    is_decoder=SCREAMING_SNAKE_CASE_   ,    initializer_range=self.initializer_range   ,    )

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      def    UpperCAmelCase						(    self			)    ->				Optional[int]:
          (
              (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,          
          )	:List[str]     							=       self.prepare_config_and_inputs()

          UpperCamelCase	:List[Any]     							=       True
          UpperCamelCase	:List[Any]     							=       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
          UpperCamelCase	:str     							=       ids_tensor([self.batch_size, self.seq_length]   ,    vocab_size=2			)

          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_			)    ->				Any:
          UpperCamelCase	:Optional[Any]     							=       RemBertModel(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Dict     							=       model(SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_			)
          UpperCamelCase	:List[Any]     							=       model(SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_			)
          UpperCamelCase	:Dict     							=       model(SCREAMING_SNAKE_CASE_			)
          self.parent.assertEqual(result.last_hidden_state.shape   ,    (self.batch_size, self.seq_length, self.hidden_size)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    )    ->				Dict:
          UpperCamelCase	:str     							=       True
          UpperCamelCase	:List[Any]     							=       RemBertModel(SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Any     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    encoder_hidden_states=SCREAMING_SNAKE_CASE_   ,    encoder_attention_mask=SCREAMING_SNAKE_CASE_   ,    )
          UpperCamelCase	:Any     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    encoder_hidden_states=SCREAMING_SNAKE_CASE_   ,    )
          UpperCamelCase	:Optional[int]     							=       model(SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_			)
          self.parent.assertEqual(result.last_hidden_state.shape   ,    (self.batch_size, self.seq_length, self.hidden_size)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    )    ->				Any:
          UpperCamelCase	:Optional[Any]     							=       RemBertForCausalLM(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Dict     							=       model(SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    labels=SCREAMING_SNAKE_CASE_			)
          self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.vocab_size)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_			)    ->				Any:
          UpperCamelCase	:Tuple     							=       RemBertForMaskedLM(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:int     							=       model(SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    labels=SCREAMING_SNAKE_CASE_			)
          self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.vocab_size)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    )    ->				List[str]:
          UpperCamelCase	:Optional[int]     							=       True
          UpperCamelCase	:Any     							=       True
          UpperCamelCase	:Tuple     							=       RemBertForCausalLM(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()

          # first forward pass
          UpperCamelCase	:Union[str, Any]     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    encoder_hidden_states=SCREAMING_SNAKE_CASE_   ,    encoder_attention_mask=SCREAMING_SNAKE_CASE_   ,    use_cache=SCREAMING_SNAKE_CASE_   ,    )
          UpperCamelCase	:List[str]     							=       outputs.past_key_values

          # create hypothetical multiple next token and extent to next_input_ids
          UpperCamelCase	:Any     							=       ids_tensor((self.batch_size, 3)   ,    config.vocab_size			)
          UpperCamelCase	:str     							=       ids_tensor((self.batch_size, 3)   ,    vocab_size=2			)

          # append to next input_ids and
          UpperCamelCase	:Tuple     							=       torch.cat([input_ids, next_tokens]   ,    dim=-1			)
          UpperCamelCase	:Dict     							=       torch.cat([input_mask, next_mask]   ,    dim=-1			)

          UpperCamelCase	:List[str]     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    encoder_hidden_states=SCREAMING_SNAKE_CASE_   ,    encoder_attention_mask=SCREAMING_SNAKE_CASE_   ,    output_hidden_states=SCREAMING_SNAKE_CASE_   ,    )['''hidden_states'''][0]
          UpperCamelCase	:Tuple     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    encoder_hidden_states=SCREAMING_SNAKE_CASE_   ,    encoder_attention_mask=SCREAMING_SNAKE_CASE_   ,    past_key_values=SCREAMING_SNAKE_CASE_   ,    output_hidden_states=SCREAMING_SNAKE_CASE_   ,    )['''hidden_states'''][0]

          # select random slice
          UpperCamelCase	:Any     							=       ids_tensor((1,)   ,    output_from_past.shape[-1]			).item()
          UpperCamelCase	:List[str]     							=       output_from_no_past[:, -3:, random_slice_idx].detach()
          UpperCamelCase	:Optional[int]     							=       output_from_past[:, :, random_slice_idx].detach()

          self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

          # test that outputs are equal for slice
          self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    atol=1e-3			)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_			)    ->				List[Any]:
          UpperCamelCase	:Union[str, Any]     							=       RemBertForQuestionAnswering(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Dict     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    start_positions=SCREAMING_SNAKE_CASE_   ,    end_positions=SCREAMING_SNAKE_CASE_   ,    )
          self.parent.assertEqual(result.start_logits.shape   ,    (self.batch_size, self.seq_length)			)
          self.parent.assertEqual(result.end_logits.shape   ,    (self.batch_size, self.seq_length)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_			)    ->				Union[str, Any]:
          UpperCamelCase	:Union[str, Any]     							=       self.num_labels
          UpperCamelCase	:List[Any]     							=       RemBertForSequenceClassification(SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Any     							=       model(SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    labels=SCREAMING_SNAKE_CASE_			)
          self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_labels)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_			)    ->				List[Any]:
          UpperCamelCase	:str     							=       self.num_labels
          UpperCamelCase	:Tuple     							=       RemBertForTokenClassification(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Optional[Any]     							=       model(SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    labels=SCREAMING_SNAKE_CASE_			)
          self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.num_labels)			)
      def    UpperCAmelCase						(    self   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_			)    ->				int:
          UpperCamelCase	:List[Any]     							=       self.num_choices
          UpperCamelCase	:List[Any]     							=       RemBertForMultipleChoice(config=SCREAMING_SNAKE_CASE_			)
          model.to(SCREAMING_SNAKE_CASE_			)
          model.eval()
          UpperCamelCase	:Optional[Any]     							=       input_ids.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
          UpperCamelCase	:Optional[int]     							=       token_type_ids.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
          UpperCamelCase	:Union[str, Any]     							=       input_mask.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
          UpperCamelCase	:Union[str, Any]     							=       model(
              SCREAMING_SNAKE_CASE_   ,    attention_mask=SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    labels=SCREAMING_SNAKE_CASE_   ,    )
          self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_choices)			)




      def    UpperCAmelCase						(    self			)    ->				Optional[Any]:
          UpperCamelCase	:Any     							=       self.prepare_config_and_inputs()
          (
              (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,          
          )	:List[str]     							=       config_and_inputs
          UpperCamelCase	:int     							=       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
          return config, inputs_dict







@require_torch
class       UpperCAmelCase_						( lowercase,     lowercase,     unittest.TestCase			):



      """simple docstring"""
      UpperCamelCase_    : Optional[int]									=(
          (
              RemBertModel,
              RemBertForMaskedLM,
              RemBertForCausalLM,
              RemBertForMultipleChoice,
              RemBertForQuestionAnswering,
              RemBertForSequenceClassification,
              RemBertForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      UpperCamelCase_    : Optional[Any]									=(RemBertForCausalLM,) if is_torch_available() else ()
      UpperCamelCase_    : Optional[Any]									=(
          {
              'feature-extraction': RemBertModel,
              'fill-mask': RemBertForMaskedLM,
              'question-answering': RemBertForQuestionAnswering,
              'text-classification': RemBertForSequenceClassification,
              'text-generation': RemBertForCausalLM,
              'token-classification': RemBertForTokenClassification,
              'zero-shot': RemBertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      def    UpperCAmelCase						(    self			)    ->				Optional[Any]:
          UpperCamelCase	:int     							=       RemBertModelTester(self			)
          UpperCamelCase	:List[Any]     							=       ConfigTester(self   ,    config_class=SCREAMING_SNAKE_CASE_   ,    hidden_size=37			)
      def    UpperCAmelCase						(    self			)    ->				Union[str, Any]:
          self.config_tester.run_common_tests()
      def    UpperCAmelCase						(    self			)    ->				Optional[int]:
          UpperCamelCase	:int     							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				List[str]:
          UpperCamelCase	:int     							=       self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
              UpperCamelCase	:Optional[int]     							=       type
              self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				int:
          UpperCamelCase	:Union[str, Any]     							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				List[Any]:
          UpperCamelCase	:str     							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				Any:
          UpperCamelCase	:List[Any]     							=       self.model_tester.prepare_config_and_inputs_for_decoder()
          self.model_tester.create_and_check_decoder_model_past_large_inputs(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				str:
          UpperCamelCase	:Dict     							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				str:
          UpperCamelCase	:str     							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				Tuple:
          UpperCamelCase	:str     							=       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				int:
          UpperCamelCase	:str     							=       self.model_tester.prepare_config_and_inputs_for_decoder()
          self.model_tester.create_and_check_model_as_decoder(*SCREAMING_SNAKE_CASE_			)
      def    UpperCAmelCase						(    self			)    ->				int:
          (
              (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,       (
              UpperCamelCase   
          )							,          
          )	:str     							=       self.model_tester.prepare_config_and_inputs_for_decoder()

          UpperCamelCase	:Union[str, Any]     							=       None

          self.model_tester.create_and_check_model_as_decoder(
              SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    SCREAMING_SNAKE_CASE_   ,    )




      @slow
      def    UpperCAmelCase						(    self			)    ->				List[Any]:



          for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              UpperCamelCase	:int     							=       RemBertModel.from_pretrained(SCREAMING_SNAKE_CASE_			)
              self.assertIsNotNone(SCREAMING_SNAKE_CASE_			)







@require_torch
class       UpperCAmelCase_						( unittest.TestCase			):



      """simple docstring"""
      @slow
      def    UpperCAmelCase						(    self			)    ->				List[Any]:
          UpperCamelCase	:List[str]     							=       RemBertModel.from_pretrained('''google/rembert'''			)
          UpperCamelCase	:List[str]     							=       torch.tensor([[312, 5_6498, 313, 2125, 313]]			)
          UpperCamelCase	:List[Any]     							=       torch.tensor([[0, 0, 0, 1, 1]]			)
          with torch.no_grad():
              UpperCamelCase	:Any     							=       model(SCREAMING_SNAKE_CASE_   ,    token_type_ids=SCREAMING_SNAKE_CASE_   ,    output_hidden_states=SCREAMING_SNAKE_CASE_			)

          UpperCamelCase	:int     							=       1152

          UpperCamelCase	:Dict     							=       torch.Size((1, 5, hidden_size)			)
          self.assertEqual(output['''last_hidden_state'''].shape   ,    SCREAMING_SNAKE_CASE_			)

          UpperCamelCase	:List[str]     							=       torch.tensor(
              [
                  [
                      [0.0754, -0.2022, 0.1904],
                      [-0.3354, -0.3692, -0.4791],
                      [-0.2314, -0.6729, -0.0749],
                      [-0.0396, -0.3105, -0.4234],
                      [-0.1571, -0.0525, 0.5353],
                  ]
              ]			)

          # Running on the original tf implementation gives slightly different results here.
          # Not clear why this variations is present
          # TODO: Find reason for discrepancy
          # expected_original_implementation = [[
          #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
          #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
          #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
          #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
          #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
          # ]]

          self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]   ,    SCREAMING_SNAKE_CASE_   ,    atol=1e-4			)			)













def   _A       (    SCREAMING_SNAKE_CASE__  :     int	):
    UpperCamelCase	:Optional[Any]     							=       9

    while num > 0:
        if num % 10 != digit:
            return False
        num //= 100
        digit -= 1

    return True


def   _A       (    ):
    UpperCamelCase	:List[str]     							=       138902663

    while not is_square_form(num * num	):
        if num % 10 == 3:
            num -= 6  # (3 - 6) % 10 = 7
        else:
            num -= 4  # (7 - 4) % 10 = 3

    return num * 10


if __name__ == "__main__":
 print(f'''{solution() = }''')





