


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      FunnelBaseModel,
      FunnelForMaskedLM,
      FunnelForMultipleChoice,
      FunnelForPreTraining,
      FunnelForQuestionAnswering,
      FunnelForSequenceClassification,
      FunnelForTokenClassification,
      FunnelModel,
  )







class 				lowerCamelCase__       :





     def __init__(self		,					UpperCAmelCase		,					UpperCAmelCase=1_3		,					UpperCAmelCase=7		,					UpperCAmelCase=True		,					UpperCAmelCase=True		,					UpperCAmelCase=True		,					UpperCAmelCase=True		,					UpperCAmelCase=9_9		,					UpperCAmelCase=[1, 1, 2]		,					UpperCAmelCase=1		,					UpperCAmelCase=3_2		,					UpperCAmelCase=4		,					UpperCAmelCase=8		,					UpperCAmelCase=3_7		,					UpperCAmelCase="gelu_new"		,					UpperCAmelCase=0.1		,					UpperCAmelCase=0.1		,					UpperCAmelCase=0.0		,					UpperCAmelCase=5_1_2		,					UpperCAmelCase=3		,					UpperCAmelCase=0.02		,					UpperCAmelCase=3		,					UpperCAmelCase=4		,					UpperCAmelCase=None		,					UpperCAmelCase=False		,					)					->       Union[str, Any]:
      _lowercase											=parent
      _lowercase											=batch_size
      _lowercase											=seq_length
      _lowercase											=is_training
      _lowercase											=use_input_mask
      _lowercase											=use_token_type_ids
      _lowercase											=use_labels
      _lowercase											=vocab_size
      _lowercase											=block_sizes
      _lowercase											=num_decoder_layers
      _lowercase											=d_model
      _lowercase											=n_head
      _lowercase											=d_head
      _lowercase											=d_inner
      _lowercase											=hidden_act
      _lowercase											=hidden_dropout
      _lowercase											=attention_dropout
      _lowercase											=activation_dropout
      _lowercase											=max_position_embeddings
      _lowercase											=type_vocab_size
      _lowercase											=2
      _lowercase											=num_labels
      _lowercase											=num_choices
      _lowercase											=scope
      _lowercase											=initializer_std

      # Used in the tests to check the size of the first attention layer
      _lowercase											=n_head
      # Used in the tests to check the size of the first hidden state
      _lowercase											=self.d_model
      # Used in the tests to check the number of output hidden states/attentions
      _lowercase											=sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
      # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
      # the last hidden state of the first block (which is the first hidden state of the decoder).
      if not base:
       _lowercase											=self.num_hidden_layers + 2





     def 					__A (self	)					->       str:
      _lowercase											=ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size	)

      _lowercase											=None
      if self.use_input_mask:
       _lowercase											=ids_tensor([self.batch_size, self.seq_length]		,					vocab_size=2	)

      _lowercase											=None
      if self.use_token_type_ids:
       _lowercase											=ids_tensor([self.batch_size, self.seq_length]		,					self.type_vocab_size	)

      _lowercase											=None
      _lowercase											=None
      _lowercase											=None
      if self.use_labels:
       _lowercase											=ids_tensor([self.batch_size]		,					self.type_sequence_label_size	)
       _lowercase											=ids_tensor([self.batch_size, self.seq_length]		,					self.num_labels	)
       _lowercase											=ids_tensor([self.batch_size]		,					self.num_choices	)
       _lowercase											=ids_tensor([self.batch_size, self.seq_length]		,					1	)

      _lowercase											=self.get_config()

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          fake_token_labels,
      )





     def 					__A (self	)					->       Optional[int]:
      return FunnelConfig(
          vocab_size=self.vocab_size		,					block_sizes=self.block_sizes		,					num_decoder_layers=self.num_decoder_layers		,					d_model=self.d_model		,					n_head=self.n_head		,					d_head=self.d_head		,					d_inner=self.d_inner		,					hidden_act=self.hidden_act		,					hidden_dropout=self.hidden_dropout		,					attention_dropout=self.attention_dropout		,					activation_dropout=self.activation_dropout		,					max_position_embeddings=self.max_position_embeddings		,					type_vocab_size=self.type_vocab_size		,					initializer_std=self.initializer_std		,					)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       Union[str, Any]:
      _lowercase											=FunnelModel(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase	)
      _lowercase											=model(__UpperCamelCase		,					token_type_ids=__UpperCamelCase	)
      _lowercase											=model(__UpperCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.d_model)	)

      _lowercase											=False
      _lowercase											=model(__UpperCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.d_model)	)

      _lowercase											=False
      _lowercase											=model(__UpperCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.d_model)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       List[str]:
      _lowercase											=FunnelBaseModel(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase	)
      _lowercase											=model(__UpperCamelCase		,					token_type_ids=__UpperCamelCase	)
      _lowercase											=model(__UpperCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, 2, self.d_model)	)

      _lowercase											=False
      _lowercase											=model(__UpperCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, 3, self.d_model)	)

      _lowercase											=False
      _lowercase											=model(__UpperCamelCase	)
      self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, 2, self.d_model)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       Tuple:
      _lowercase											=self.num_labels
      _lowercase											=FunnelForPreTraining(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase		,					labels=__UpperCamelCase	)
      self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       List[Any]:
      _lowercase											=FunnelForMaskedLM(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase		,					labels=__UpperCamelCase	)
      self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       Optional[int]:
      _lowercase											=self.num_labels
      _lowercase											=FunnelForSequenceClassification(__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase		,					labels=__UpperCamelCase	)
      self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_labels)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       str:
      _lowercase											=self.num_choices
      _lowercase											=FunnelForMultipleChoice(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=input_ids.unsqueeze(1	).expand(-1		,					self.num_choices		,					-1	).contiguous()
      _lowercase											=token_type_ids.unsqueeze(1	).expand(-1		,					self.num_choices		,					-1	).contiguous()
      _lowercase											=input_mask.unsqueeze(1	).expand(-1		,					self.num_choices		,					-1	).contiguous()
      _lowercase											=model(
          __UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase		,					labels=__UpperCamelCase		,					)
      self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_choices)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       int:
      _lowercase											=self.num_labels
      _lowercase											=FunnelForTokenClassification(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase		,					labels=__UpperCamelCase	)
      self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.num_labels)	)





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase		,					)					->       Any:
      _lowercase											=FunnelForQuestionAnswering(config=__UpperCamelCase	)
      model.to(__UpperCamelCase	)
      model.eval()
      _lowercase											=model(
          __UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase		,					start_positions=__UpperCamelCase		,					end_positions=__UpperCamelCase		,					)
      self.parent.assertEqual(result.start_logits.shape		,					(self.batch_size, self.seq_length)	)
      self.parent.assertEqual(result.end_logits.shape		,					(self.batch_size, self.seq_length)	)





     def 					__A (self	)					->       Optional[Any]:
      _lowercase											=self.prepare_config_and_inputs()
      (
          (
          _lowercase     
      )				,   (
          _lowercase     
      )				,   (
          _lowercase     
      )				,   (
          _lowercase     
      )				,   (
          _lowercase     
      )				,   (
          _lowercase     
      )				,   (
          _lowercase     
      )				,   (
          _lowercase     
      )				,        
      )											=config_and_inputs
      _lowercase											={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
      return config, inputs_dict







@require_torch
class 				lowerCamelCase__       (	lowercase_       ,							lowercase_       ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__				      =      False
     SCREAMING_SNAKE_CASE__				      =      False
     SCREAMING_SNAKE_CASE__				      =      (
         (
             FunnelModel,
             FunnelForMaskedLM,
             FunnelForPreTraining,
             FunnelForQuestionAnswering,
             FunnelForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     SCREAMING_SNAKE_CASE__				      =      (
         {
             '''feature-extraction''': (FunnelBaseModel, FunnelModel),
             '''fill-mask''': FunnelForMaskedLM,
             '''question-answering''': FunnelForQuestionAnswering,
             '''text-classification''': FunnelForSequenceClassification,
             '''token-classification''': FunnelForTokenClassification,
             '''zero-shot''': FunnelForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )





     def 					__A (self		,					UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase=False	)					->       Optional[Any]:
      _lowercase											=super()._prepare_for_class(__UpperCamelCase		,					__UpperCamelCase		,					return_labels=__UpperCamelCase	)

      if return_labels:
       if model_class in get_values(__UpperCamelCase	):
        _lowercase											=torch.zeros(
            (self.model_tester.batch_size, self.model_tester.seq_length)		,					dtype=torch.long		,					device=__UpperCamelCase	)
      return inputs_dict





     def 					__A (self	)					->       Dict:
      _lowercase											=FunnelModelTester(self	)
      _lowercase											=ConfigTester(self		,					config_class=__UpperCamelCase	)





     def 					__A (self	)					->       int:
      self.config_tester.run_common_tests()





     def 					__A (self	)					->       List[str]:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__UpperCamelCase	)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_pretraining(*__UpperCamelCase	)





     def 					__A (self	)					->       Any:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*__UpperCamelCase	)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*__UpperCamelCase	)





     def 					__A (self	)					->       Tuple:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*__UpperCamelCase	)





     def 					__A (self		,					UpperCAmelCase	)					->       Union[str, Any]:
      if hasattr(__UpperCamelCase		,					'''weight'''	) and module.weight is not None:
       module.weight.data.fill_(3	)
      if hasattr(__UpperCamelCase		,					'''bias'''	) and module.bias is not None:
       module.bias.data.fill_(3	)

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__UpperCamelCase		,					__UpperCamelCase	) and getattr(__UpperCamelCase		,					__UpperCamelCase	) is not None:
        _lowercase											=getattr(__UpperCamelCase		,					__UpperCamelCase	)
        weight.data.fill_(3	)







@require_torch
class 				lowerCamelCase__       (	lowercase_       ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__				      =      False
     SCREAMING_SNAKE_CASE__				      =      False
     SCREAMING_SNAKE_CASE__				      =      (
         (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
     )





     def 					__A (self	)					->       Tuple:
      _lowercase											=FunnelModelTester(self		,					base=__UpperCamelCase	)
      _lowercase											=ConfigTester(self		,					config_class=__UpperCamelCase	)





     def 					__A (self	)					->       Dict:
      self.config_tester.run_common_tests()





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_base_model(*__UpperCamelCase	)





     def 					__A (self	)					->       List[Any]:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*__UpperCamelCase	)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase											=self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*__UpperCamelCase	)





     def 					__A (self	)					->       Union[str, Any]:
      _lowercase				,   _lowercase											=self.model_tester.prepare_config_and_inputs_for_common()
      _lowercase											=True

      for model_class in self.all_model_classes:
       if model_class.__name__ == "FunnelBaseModel":
        continue
       _lowercase											=model_class(__UpperCamelCase	)
       model.to(__UpperCamelCase	)
       model.train()
       _lowercase											=self._prepare_for_class(__UpperCamelCase		,					__UpperCamelCase		,					return_labels=__UpperCamelCase	)
       _lowercase											=model(**__UpperCamelCase	).loss
       loss.backward()





     def 					__A (self		,					UpperCAmelCase	)					->       Optional[Any]:
      if hasattr(__UpperCamelCase		,					'''weight'''	) and module.weight is not None:
       module.weight.data.fill_(3	)
      if hasattr(__UpperCamelCase		,					'''bias'''	) and module.bias is not None:
       module.bias.data.fill_(3	)

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__UpperCamelCase		,					__UpperCamelCase	) and getattr(__UpperCamelCase		,					__UpperCamelCase	) is not None:
        _lowercase											=getattr(__UpperCamelCase		,					__UpperCamelCase	)
        weight.data.fill_(3	)







@require_torch
@require_sentencepiece
@require_tokenizers
class 				lowerCamelCase__       (	unittest.TestCase):





     def 					__A (self	)					->       Tuple:
      _lowercase											=1_3
      _lowercase											=7
      _lowercase											=torch.arange(0		,					batch_size * sequence_length	).long().reshape(__UpperCamelCase		,					__UpperCamelCase	)
      _lowercase											=[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
      _lowercase											=torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]	)

      _lowercase											=FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''	)
      _lowercase											=model(__UpperCamelCase		,					token_type_ids=__UpperCamelCase	)[0].abs()

      _lowercase											=torch.tensor(2_3_4_4.8_3_5_2	)
      _lowercase											=torch.tensor(0.8052	)
      self.assertTrue(torch.allclose(output.sum()		,					__UpperCamelCase		,					atol=1e-4	)	)
      self.assertTrue(torch.allclose(output.mean()		,					__UpperCamelCase		,					atol=1e-4	)	)

      _lowercase											=torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]	)
      _lowercase											=model(__UpperCamelCase		,					attention_mask=__UpperCamelCase		,					token_type_ids=__UpperCamelCase	)[0].abs()

      _lowercase											=torch.tensor(2_3_4_3.8_4_2_5	)
      _lowercase											=torch.tensor(0.8049	)
      self.assertTrue(torch.allclose(output.sum()		,					__UpperCamelCase		,					atol=1e-4	)	)
      self.assertTrue(torch.allclose(output.mean()		,					__UpperCamelCase		,					atol=1e-4	)	)





     @slow
     def 					__A (self	)					->       Optional[Any]:
      _lowercase											=FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''	)
      _lowercase											=FunnelModel.from_pretrained('''huggingface/funnel-small'''	)
      _lowercase											=tokenizer('''Hello! I am the Funnel Transformer model.'''		,					return_tensors='''pt'''	)
      _lowercase											=model(**__UpperCamelCase	)[0]

      _lowercase											=torch.tensor(235.7246	)
      _lowercase											=torch.tensor(0.0256	)
      self.assertTrue(torch.allclose(output.sum()		,					__UpperCamelCase		,					atol=1e-4	)	)
      self.assertTrue(torch.allclose(output.mean()		,					__UpperCamelCase		,					atol=1e-4	)	)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__            =							logging.get_logger(__name__)

UpperCAmelCase__            =							{
    '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/config.json''',
    # QDQBERT models can be loaded from any BERT checkpoint, available at https://huggingface.co/models?filter=bert
}







class 				lowerCamelCase__       (	lowerCAmelCase):
     SCREAMING_SNAKE_CASE__				      =      '''qdqbert'''





     def __init__(self		,					UpperCAmelCase=3_0_5_2_2		,					UpperCAmelCase=7_6_8		,					UpperCAmelCase=1_2		,					UpperCAmelCase=1_2		,					UpperCAmelCase=3_0_7_2		,					UpperCAmelCase="gelu"		,					UpperCAmelCase=0.1		,					UpperCAmelCase=0.1		,					UpperCAmelCase=5_1_2		,					UpperCAmelCase=2		,					UpperCAmelCase=0.02		,					UpperCAmelCase=1e-12		,					UpperCAmelCase=True		,					UpperCAmelCase=1		,					UpperCAmelCase=0		,					UpperCAmelCase=2		,					**UpperCAmelCase		,					)					->       str:
      super().__init__(pad_token_id=UpperCAmelCase		,					bos_token_id=UpperCAmelCase		,					eos_token_id=UpperCAmelCase		,					**UpperCAmelCase	)

      _lowercase											=vocab_size
      _lowercase											=max_position_embeddings
      _lowercase											=hidden_size
      _lowercase											=num_hidden_layers
      _lowercase											=num_attention_heads
      _lowercase											=intermediate_size
      _lowercase											=hidden_act
      _lowercase											=hidden_dropout_prob
      _lowercase											=attention_probs_dropout_prob
      _lowercase											=initializer_range
      _lowercase											=type_vocab_size
      _lowercase											=layer_norm_eps
      _lowercase											=use_cache

