


'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RemBertConfig,
      RemBertForCausalLM,
      RemBertForMaskedLM,
      RemBertForMultipleChoice,
      RemBertForQuestionAnswering,
      RemBertForSequenceClassification,
      RemBertForTokenClassification,
      RemBertModel,
  )
  from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class  lowercase_					:


    def __init__(					self  , a  , a=13  , a=7  , a=True  , a=True  , a=True  , a=True  , a=99  , a=32  , a=18  , a=43  , a=5  , a=4  , a=37  , a="gelu"  , a=0.1  , a=0.1  , a=5_12  , a=16  , a=2  , a=0.02  , a=3  , a=4  , a=None  , ):
          UpperCamelCase__  =    parent
          UpperCamelCase__  =    batch_size
          UpperCamelCase__  =    seq_length
          UpperCamelCase__  =    is_training
          UpperCamelCase__  =    use_input_mask
          UpperCamelCase__  =    use_token_type_ids
          UpperCamelCase__  =    use_labels
          UpperCamelCase__  =    vocab_size
          UpperCamelCase__  =    hidden_size
          UpperCamelCase__  =    input_embedding_size
          UpperCamelCase__  =    output_embedding_size
          UpperCamelCase__  =    num_hidden_layers
          UpperCamelCase__  =    num_attention_heads
          UpperCamelCase__  =    intermediate_size
          UpperCamelCase__  =    hidden_act
          UpperCamelCase__  =    hidden_dropout_prob
          UpperCamelCase__  =    attention_probs_dropout_prob
          UpperCamelCase__  =    max_position_embeddings
          UpperCamelCase__  =    type_vocab_size
          UpperCamelCase__  =    type_sequence_label_size
          UpperCamelCase__  =    initializer_range
          UpperCamelCase__  =    num_labels
          UpperCamelCase__  =    num_choices
          UpperCamelCase__  =    scope


    def        __a							(					self   ):
          UpperCamelCase__  =    ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size   )

          UpperCamelCase__  =    None
          if self.use_input_mask:
                UpperCamelCase__  =    random_attention_mask([self.batch_size, self.seq_length]   )

          UpperCamelCase__  =    None
          if self.use_token_type_ids:
                UpperCamelCase__  =    ids_tensor([self.batch_size, self.seq_length]  , self.type_vocab_size   )

          UpperCamelCase__  =    None
          UpperCamelCase__  =    None
          UpperCamelCase__  =    None
          if self.use_labels:
                UpperCamelCase__  =    ids_tensor([self.batch_size]  , self.type_sequence_label_size   )
                UpperCamelCase__  =    ids_tensor([self.batch_size, self.seq_length]  , self.num_labels   )
                UpperCamelCase__  =    ids_tensor([self.batch_size]  , self.num_choices   )

          UpperCamelCase__  =    RemBertConfig(
              vocab_size=self.vocab_size  , hidden_size=self.hidden_size  , input_embedding_size=self.input_embedding_size  , output_embedding_size=self.output_embedding_size  , num_hidden_layers=self.num_hidden_layers  , num_attention_heads=self.num_attention_heads  , intermediate_size=self.intermediate_size  , hidden_act=self.hidden_act  , hidden_dropout_prob=self.hidden_dropout_prob  , attention_probs_dropout_prob=self.attention_probs_dropout_prob  , max_position_embeddings=self.max_position_embeddings  , type_vocab_size=self.type_vocab_size  , is_decoder=_A  , initializer_range=self.initializer_range  , )

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


    def        __a							(					self   ):
          (
              UpperCamelCase__						
          )  =    self.prepare_config_and_inputs()

          UpperCamelCase__  =    True
          UpperCamelCase__  =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
          UpperCamelCase__  =    ids_tensor([self.batch_size, self.seq_length]  , vocab_size=2   )

          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    RemBertModel(config=_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(_A  , attention_mask=_A  , token_type_ids=_A   )
          UpperCamelCase__  =    model(_A  , token_type_ids=_A   )
          UpperCamelCase__  =    model(_A   )
          self.parent.assertEqual(result.last_hidden_state.shape  , (self.batch_size, self.seq_length, self.hidden_size)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a  , a  , a  , ):
          UpperCamelCase__  =    True
          UpperCamelCase__  =    RemBertModel(_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , token_type_ids=_A  , encoder_hidden_states=_A  , encoder_attention_mask=_A  , )
          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , token_type_ids=_A  , encoder_hidden_states=_A  , )
          UpperCamelCase__  =    model(_A  , attention_mask=_A  , token_type_ids=_A   )
          self.parent.assertEqual(result.last_hidden_state.shape  , (self.batch_size, self.seq_length, self.hidden_size)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a  , a  , a  , ):
          UpperCamelCase__  =    RemBertForCausalLM(config=_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(_A  , attention_mask=_A  , token_type_ids=_A  , labels=_A   )
          self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.vocab_size)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    RemBertForMaskedLM(config=_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(_A  , attention_mask=_A  , token_type_ids=_A  , labels=_A   )
          self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.vocab_size)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a  , a  , a  , ):
          UpperCamelCase__  =    True
          UpperCamelCase__  =    True
          UpperCamelCase__  =    RemBertForCausalLM(config=_A   )
          model.to(_A   )
          model.eval()

          # first forward pass
          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , encoder_hidden_states=_A  , encoder_attention_mask=_A  , use_cache=_A  , )
          UpperCamelCase__  =    outputs.past_key_values

          # create hypothetical multiple next token and extent to next_input_ids
          UpperCamelCase__  =    ids_tensor((self.batch_size, 3)  , config.vocab_size   )
          UpperCamelCase__  =    ids_tensor((self.batch_size, 3)  , vocab_size=2   )

          # append to next input_ids and
          UpperCamelCase__  =    torch.cat([input_ids, next_tokens]  , dim=-1   )
          UpperCamelCase__  =    torch.cat([input_mask, next_mask]  , dim=-1   )

          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , encoder_hidden_states=_A  , encoder_attention_mask=_A  , output_hidden_states=_A  , )['''hidden_states'''][0]
          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , encoder_hidden_states=_A  , encoder_attention_mask=_A  , past_key_values=_A  , output_hidden_states=_A  , )['''hidden_states'''][0]

          # select random slice
          UpperCamelCase__  =    ids_tensor((1,)  , output_from_past.shape[-1]   ).item()
          UpperCamelCase__  =    output_from_no_past[:, -3:, random_slice_idx].detach()
          UpperCamelCase__  =    output_from_past[:, :, random_slice_idx].detach()

          self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

          # test that outputs are equal for slice
          self.parent.assertTrue(torch.allclose(_A  , _A  , atol=1e-3   )   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    RemBertForQuestionAnswering(config=_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , token_type_ids=_A  , start_positions=_A  , end_positions=_A  , )
          self.parent.assertEqual(result.start_logits.shape  , (self.batch_size, self.seq_length)   )
          self.parent.assertEqual(result.end_logits.shape  , (self.batch_size, self.seq_length)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    self.num_labels
          UpperCamelCase__  =    RemBertForSequenceClassification(_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(_A  , attention_mask=_A  , token_type_ids=_A  , labels=_A   )
          self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.num_labels)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    self.num_labels
          UpperCamelCase__  =    RemBertForTokenClassification(config=_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    model(_A  , attention_mask=_A  , token_type_ids=_A  , labels=_A   )
          self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.num_labels)   )


    def        __a							(					self  , a  , a  , a  , a  , a  , a  , a   ):
          UpperCamelCase__  =    self.num_choices
          UpperCamelCase__  =    RemBertForMultipleChoice(config=_A   )
          model.to(_A   )
          model.eval()
          UpperCamelCase__  =    input_ids.unsqueeze(1   ).expand(-1  , self.num_choices  , -1   ).contiguous()
          UpperCamelCase__  =    token_type_ids.unsqueeze(1   ).expand(-1  , self.num_choices  , -1   ).contiguous()
          UpperCamelCase__  =    input_mask.unsqueeze(1   ).expand(-1  , self.num_choices  , -1   ).contiguous()
          UpperCamelCase__  =    model(
              _A  , attention_mask=_A  , token_type_ids=_A  , labels=_A  , )
          self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.num_choices)   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.prepare_config_and_inputs()
          (
              UpperCamelCase__						
          )  =    config_and_inputs
          UpperCamelCase__  =    {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
          return config, inputs_dict

@require_torch
class  lowercase_					(   lowerCAmelCase__  ,						lowerCAmelCase__  ,						unittest.TestCase       ):
    __UpperCAmelCase             =     (
        (
            RemBertModel,
            RemBertForMaskedLM,
            RemBertForCausalLM,
            RemBertForMultipleChoice,
            RemBertForQuestionAnswering,
            RemBertForSequenceClassification,
            RemBertForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    __UpperCAmelCase             =     (RemBertForCausalLM,) if is_torch_available() else ()
    __UpperCAmelCase             =     (
        {
            'feature-extraction': RemBertModel,
            'fill-mask': RemBertForMaskedLM,
            'question-answering': RemBertForQuestionAnswering,
            'text-classification': RemBertForSequenceClassification,
            'text-generation': RemBertForCausalLM,
            'token-classification': RemBertForTokenClassification,
            'zero-shot': RemBertForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )


    def        __a							(					self   ):
          UpperCamelCase__  =    RemBertModelTester(self   )
          UpperCamelCase__  =    ConfigTester(self  , config_class=_A  , hidden_size=37   )


    def        __a							(					self   ):
          self.config_tester.run_common_tests()


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
                UpperCamelCase__  =    type
                self.model_tester.create_and_check_model(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs_for_decoder()
          self.model_tester.create_and_check_decoder_model_past_large_inputs(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_A   )


    def        __a							(					self   ):
          UpperCamelCase__  =    self.model_tester.prepare_config_and_inputs_for_decoder()
          self.model_tester.create_and_check_model_as_decoder(*_A   )


    def        __a							(					self   ):
          (
              UpperCamelCase__						
          )  =    self.model_tester.prepare_config_and_inputs_for_decoder()

          UpperCamelCase__  =    None

          self.model_tester.create_and_check_model_as_decoder(
              _A  , _A  , _A  , _A  , _A  , _A  , _A  , _A  , _A  , )


    @slow
    def        __a							(					self   ):




          for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                UpperCamelCase__  =    RemBertModel.from_pretrained(_A   )
                self.assertIsNotNone(_A   )

@require_torch
class  lowercase_					(   unittest.TestCase       ):


    @slow
    def        __a							(					self   ):
          UpperCamelCase__  =    RemBertModel.from_pretrained("google/rembert"   )
          UpperCamelCase__  =    torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]   )
          UpperCamelCase__  =    torch.tensor([[0, 0, 0, 1, 1]]   )
          with torch.no_grad():
                UpperCamelCase__  =    model(_A  , token_type_ids=_A  , output_hidden_states=_A   )

          UpperCamelCase__  =    11_52

          UpperCamelCase__  =    torch.Size((1, 5, hidden_size)   )
          self.assertEqual(output["last_hidden_state"].shape  , _A   )

          UpperCamelCase__  =    torch.tensor(
              [
                  [
                      [0.0754, -0.2022, 0.1904],
                      [-0.3354, -0.3692, -0.4791],
                      [-0.2314, -0.6729, -0.0749],
                      [-0.0396, -0.3105, -0.4234],
                      [-0.1571, -0.0525, 0.5353],
                  ]
              ]   )

          # Running on the original tf implementation gives slightly different results here.
          # Not clear why this variations is present
          # TODO: Find reason for discrepancy
          # expected_original_implementation = [[
          #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
          #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
          #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
          #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
          #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
          # ]]

          self.assertTrue(torch.allclose(output["last_hidden_state"][:, :, :3]  , _A  , atol=1e-4   )   )










'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a__  : Any				=	{
    'configuration_codegen': ['CODEGEN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'CodeGenConfig', 'CodeGenOnnxConfig'],
    'tokenization_codegen': ['CodeGenTokenizer'],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a__  : Tuple				=	['CodeGenTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a__  : Dict				=	[
      'CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST',
      'CodeGenForCausalLM',
      'CodeGenModel',
      'CodeGenPreTrainedModel',
  ]

if TYPE_CHECKING:
  from .configuration_codegen import CODEGEN_PRETRAINED_CONFIG_ARCHIVE_MAP, CodeGenConfig, CodeGenOnnxConfig
  from .tokenization_codegen import CodeGenTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_codegen_fast import CodeGenTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_codegen import (
        CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST,
        CodeGenForCausalLM,
        CodeGenModel,
        CodeGenPreTrainedModel,
    )

else:
  import sys

  a__  : Optional[int]				=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





