





'''simple docstring'''




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RemBertConfig,
							    RemBertForCausalLM,
							    RemBertForMaskedLM,
							    RemBertForMultipleChoice,
							    RemBertForQuestionAnswering,
							    RemBertForSequenceClassification,
							    RemBertForTokenClassification,
							    RemBertModel,
							)
							from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class       UpperCamelCase__			:




					"""simple docstring"""


					def __init__(		self							:	str     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	List[Any]=13     ,		lowerCamelCase_							:	Dict=7     ,		lowerCamelCase_							:	Tuple=True     ,		lowerCamelCase_							:	Tuple=True     ,		lowerCamelCase_							:	Any=True     ,		lowerCamelCase_							:	Union[str, Any]=True     ,		lowerCamelCase_							:	int=99     ,		lowerCamelCase_							:	Dict=32     ,		lowerCamelCase_							:	Union[str, Any]=18     ,		lowerCamelCase_							:	Dict=43     ,		lowerCamelCase_							:	str=5     ,		lowerCamelCase_							:	Dict=4     ,		lowerCamelCase_							:	int=37     ,		lowerCamelCase_							:	Optional[int]="gelu"     ,		lowerCamelCase_							:	Dict=0.1     ,		lowerCamelCase_							:	Optional[int]=0.1     ,		lowerCamelCase_							:	int=5_12     ,		lowerCamelCase_							:	Optional[int]=16     ,		lowerCamelCase_							:	List[str]=2     ,		lowerCamelCase_							:	Optional[Any]=0.02     ,		lowerCamelCase_							:	int=3     ,		lowerCamelCase_							:	Tuple=4     ,		lowerCamelCase_							:	Optional[int]=None     ,		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							List[Any]						=	parent
							SCREAMING_SNAKE_CASE				:							Dict						=	batch_size
							SCREAMING_SNAKE_CASE				:							List[Any]						=	seq_length
							SCREAMING_SNAKE_CASE				:							List[Any]						=	is_training
							SCREAMING_SNAKE_CASE				:							int						=	use_input_mask
							SCREAMING_SNAKE_CASE				:							List[str]						=	use_token_type_ids
							SCREAMING_SNAKE_CASE				:							str						=	use_labels
							SCREAMING_SNAKE_CASE				:							Any						=	vocab_size
							SCREAMING_SNAKE_CASE				:							Dict						=	hidden_size
							SCREAMING_SNAKE_CASE				:							List[str]						=	input_embedding_size
							SCREAMING_SNAKE_CASE				:							str						=	output_embedding_size
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	num_hidden_layers
							SCREAMING_SNAKE_CASE				:							List[Any]						=	num_attention_heads
							SCREAMING_SNAKE_CASE				:							Dict						=	intermediate_size
							SCREAMING_SNAKE_CASE				:							List[Any]						=	hidden_act
							SCREAMING_SNAKE_CASE				:							Optional[int]						=	hidden_dropout_prob
							SCREAMING_SNAKE_CASE				:							List[Any]						=	attention_probs_dropout_prob
							SCREAMING_SNAKE_CASE				:							Dict						=	max_position_embeddings
							SCREAMING_SNAKE_CASE				:							Optional[int]						=	type_vocab_size
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	type_sequence_label_size
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	initializer_range
							SCREAMING_SNAKE_CASE				:							List[str]						=	num_labels
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	num_choices
							SCREAMING_SNAKE_CASE				:							Dict						=	scope


					def lowerCamelCase_	(		self							:	List[str]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Tuple						=	ids_tensor([self.batch_size, self.seq_length]     ,		self.vocab_size		)

							SCREAMING_SNAKE_CASE				:							List[Any]						=	None
							if self.use_input_mask:
									SCREAMING_SNAKE_CASE				:							List[Any]						=	random_attention_mask([self.batch_size, self.seq_length]		)

							SCREAMING_SNAKE_CASE				:							Dict						=	None
							if self.use_token_type_ids:
									SCREAMING_SNAKE_CASE				:							str						=	ids_tensor([self.batch_size, self.seq_length]     ,		self.type_vocab_size		)

							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	None
							SCREAMING_SNAKE_CASE				:							int						=	None
							SCREAMING_SNAKE_CASE				:							int						=	None
							if self.use_labels:
									SCREAMING_SNAKE_CASE				:							List[str]						=	ids_tensor([self.batch_size]     ,		self.type_sequence_label_size		)
									SCREAMING_SNAKE_CASE				:							int						=	ids_tensor([self.batch_size, self.seq_length]     ,		self.num_labels		)
									SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	ids_tensor([self.batch_size]     ,		self.num_choices		)

							SCREAMING_SNAKE_CASE				:							str						=	RemBertConfig(
							    vocab_size=self.vocab_size     ,		hidden_size=self.hidden_size     ,		input_embedding_size=self.input_embedding_size     ,		output_embedding_size=self.output_embedding_size     ,		num_hidden_layers=self.num_hidden_layers     ,		num_attention_heads=self.num_attention_heads     ,		intermediate_size=self.intermediate_size     ,		hidden_act=self.hidden_act     ,		hidden_dropout_prob=self.hidden_dropout_prob     ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,		max_position_embeddings=self.max_position_embeddings     ,		type_vocab_size=self.type_vocab_size     ,		is_decoder=lowerCamelCase_     ,		initializer_range=self.initializer_range     ,		)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


					def lowerCamelCase_	(		self							:	Union[str, Any]		):

							'''simple docstring'''



							(
							    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    
							)				:							List[str]						=	self.prepare_config_and_inputs()

							SCREAMING_SNAKE_CASE				:							int						=	True
							SCREAMING_SNAKE_CASE				:							str						=	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
							SCREAMING_SNAKE_CASE				:							List[str]						=	ids_tensor([self.batch_size, self.seq_length]     ,		vocab_size=2		)

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    encoder_hidden_states,
							    encoder_attention_mask,
							)


					def lowerCamelCase_	(		self							:	List[str]     ,		lowerCamelCase_							:	Optional[Any]     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	List[Any]     ,		lowerCamelCase_							:	List[str]     ,		lowerCamelCase_							:	Any		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							List[Any]						=	RemBertModel(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							List[Any]						=	model(lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_		)
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	model(lowerCamelCase_     ,		token_type_ids=lowerCamelCase_		)
							SCREAMING_SNAKE_CASE				:							Tuple						=	model(lowerCamelCase_		)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.hidden_size)		)


					def lowerCamelCase_	(		self							:	Optional[Any]     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	Optional[Any]     ,		lowerCamelCase_							:	List[str]     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	List[Any]     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	List[str]     ,		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Any						=	True
							SCREAMING_SNAKE_CASE				:							Dict						=	RemBertModel(lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							int						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		encoder_hidden_states=lowerCamelCase_     ,		encoder_attention_mask=lowerCamelCase_     ,		)
							SCREAMING_SNAKE_CASE				:							List[Any]						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		encoder_hidden_states=lowerCamelCase_     ,		)
							SCREAMING_SNAKE_CASE				:							Tuple						=	model(lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_		)
							self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.hidden_size)		)


					def lowerCamelCase_	(		self							:	str     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	int     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	List[str]     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	int     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	Optional[int]     ,		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							int						=	RemBertForCausalLM(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							Optional[int]						=	model(lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		labels=lowerCamelCase_		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.vocab_size)		)


					def lowerCamelCase_	(		self							:	Optional[int]     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	List[Any]     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	Any		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							str						=	RemBertForMaskedLM(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							Any						=	model(lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		labels=lowerCamelCase_		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.vocab_size)		)


					def lowerCamelCase_	(		self							:	List[Any]     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	str     ,		lowerCamelCase_							:	List[str]     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	int     ,		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	True
							SCREAMING_SNAKE_CASE				:							Any						=	True
							SCREAMING_SNAKE_CASE				:							List[str]						=	RemBertForCausalLM(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()

							# first forward pass
							SCREAMING_SNAKE_CASE				:							int						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		encoder_hidden_states=lowerCamelCase_     ,		encoder_attention_mask=lowerCamelCase_     ,		use_cache=lowerCamelCase_     ,		)
							SCREAMING_SNAKE_CASE				:							Any						=	outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							SCREAMING_SNAKE_CASE				:							Any						=	ids_tensor((self.batch_size, 3)     ,		config.vocab_size		)
							SCREAMING_SNAKE_CASE				:							List[str]						=	ids_tensor((self.batch_size, 3)     ,		vocab_size=2		)

							# append to next input_ids and
							SCREAMING_SNAKE_CASE				:							Dict						=	torch.cat([input_ids, next_tokens]     ,		dim=-1		)
							SCREAMING_SNAKE_CASE				:							int						=	torch.cat([input_mask, next_mask]     ,		dim=-1		)

							SCREAMING_SNAKE_CASE				:							Dict						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		encoder_hidden_states=lowerCamelCase_     ,		encoder_attention_mask=lowerCamelCase_     ,		output_hidden_states=lowerCamelCase_     ,		)["""hidden_states"""][0]
							SCREAMING_SNAKE_CASE				:							int						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		encoder_hidden_states=lowerCamelCase_     ,		encoder_attention_mask=lowerCamelCase_     ,		past_key_values=lowerCamelCase_     ,		output_hidden_states=lowerCamelCase_     ,		)["""hidden_states"""][0]

							# select random slice
							SCREAMING_SNAKE_CASE				:							List[str]						=	ids_tensor((1,)     ,		output_from_past.shape[-1]		).item()
							SCREAMING_SNAKE_CASE				:							List[Any]						=	output_from_no_past[:, -3:, random_slice_idx].detach()
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(lowerCamelCase_     ,		lowerCamelCase_     ,		atol=1e-3		)		)


					def lowerCamelCase_	(		self							:	List[Any]     ,		lowerCamelCase_							:	List[Any]     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	int     ,		lowerCamelCase_							:	str     ,		lowerCamelCase_							:	List[Any]     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	List[str]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							List[str]						=	RemBertForQuestionAnswering(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		start_positions=lowerCamelCase_     ,		end_positions=lowerCamelCase_     ,		)
							self.parent.assertEqual(result.start_logits.shape     ,		(self.batch_size, self.seq_length)		)
							self.parent.assertEqual(result.end_logits.shape     ,		(self.batch_size, self.seq_length)		)


					def lowerCamelCase_	(		self							:	Tuple     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	str     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	Optional[Any]     ,		lowerCamelCase_							:	Tuple     ,		lowerCamelCase_							:	int     ,		lowerCamelCase_							:	str		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							List[str]						=	self.num_labels
							SCREAMING_SNAKE_CASE				:							Any						=	RemBertForSequenceClassification(lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							str						=	model(lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		labels=lowerCamelCase_		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_labels)		)


					def lowerCamelCase_	(		self							:	Any     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	str     ,		lowerCamelCase_							:	int     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	Dict		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Dict						=	self.num_labels
							SCREAMING_SNAKE_CASE				:							int						=	RemBertForTokenClassification(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							str						=	model(lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		labels=lowerCamelCase_		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.seq_length, self.num_labels)		)


					def lowerCamelCase_	(		self							:	Tuple     ,		lowerCamelCase_							:	Optional[Any]     ,		lowerCamelCase_							:	Any     ,		lowerCamelCase_							:	Dict     ,		lowerCamelCase_							:	Union[str, Any]     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	Optional[int]     ,		lowerCamelCase_							:	Tuple		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							List[Any]						=	self.num_choices
							SCREAMING_SNAKE_CASE				:							Any						=	RemBertForMultipleChoice(config=lowerCamelCase_		)
							model.to(lowerCamelCase_		)
							model.eval()
							SCREAMING_SNAKE_CASE				:							List[Any]						=	input_ids.unsqueeze(1		).expand(-1     ,		self.num_choices     ,		-1		).contiguous()
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	token_type_ids.unsqueeze(1		).expand(-1     ,		self.num_choices     ,		-1		).contiguous()
							SCREAMING_SNAKE_CASE				:							List[Any]						=	input_mask.unsqueeze(1		).expand(-1     ,		self.num_choices     ,		-1		).contiguous()
							SCREAMING_SNAKE_CASE				:							Dict						=	model(
							    lowerCamelCase_     ,		attention_mask=lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		labels=lowerCamelCase_     ,		)
							self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_choices)		)



					def lowerCamelCase_	(		self							:	Dict		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self.prepare_config_and_inputs()
							(
							    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    
							)				:							Dict						=	config_and_inputs
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
							return config, inputs_dict







@require_torch
class       UpperCamelCase__			(			lowercase_		,			lowercase_		,			unittest.TestCase	):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__						=					(
					    (
					        RemBertModel,
					        RemBertForMaskedLM,
					        RemBertForCausalLM,
					        RemBertForMultipleChoice,
					        RemBertForQuestionAnswering,
					        RemBertForSequenceClassification,
					        RemBertForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					SCREAMING_SNAKE_CASE__						=					(RemBertForCausalLM,) if is_torch_available() else ()
					SCREAMING_SNAKE_CASE__						=					(
					    {
					        '''feature-extraction''': RemBertModel,
					        '''fill-mask''': RemBertForMaskedLM,
					        '''question-answering''': RemBertForQuestionAnswering,
					        '''text-classification''': RemBertForSequenceClassification,
					        '''text-generation''': RemBertForCausalLM,
					        '''token-classification''': RemBertForTokenClassification,
					        '''zero-shot''': RemBertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)


					def lowerCamelCase_	(		self							:	Tuple		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	RemBertModelTester(self		)
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	ConfigTester(self     ,		config_class=lowerCamelCase_     ,		hidden_size=37		)


					def lowerCamelCase_	(		self							:	Optional[int]		):

							'''simple docstring'''



							self.config_tester.run_common_tests()


					def lowerCamelCase_	(		self							:	Tuple		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Dict						=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	Dict		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Dict						=	self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
									SCREAMING_SNAKE_CASE				:							str						=	type
									self.model_tester.create_and_check_model(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	List[str]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	str		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	str		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Dict						=	self.model_tester.prepare_config_and_inputs_for_decoder()
							self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	List[Any]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							List[str]						=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	Union[str, Any]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Dict						=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	Optional[int]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Any						=	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	Optional[Any]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	self.model_tester.prepare_config_and_inputs_for_decoder()
							self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase_		)


					def lowerCamelCase_	(		self							:	List[Any]		):

							'''simple docstring'''



							(
							    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    (
							    SCREAMING_SNAKE_CASE
							),    
							)				:							List[Any]						=	self.model_tester.prepare_config_and_inputs_for_decoder()

							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	None

							self.model_tester.create_and_check_model_as_decoder(
							    lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_     ,		)



					@slow
					def lowerCamelCase_	(		self							:	Optional[int]		):

							'''simple docstring'''



							for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE				:							List[Any]						=	RemBertModel.from_pretrained(lowerCamelCase_		)
									self.assertIsNotNone(lowerCamelCase_		)







@require_torch
class       UpperCamelCase__			(			unittest.TestCase	):




					"""simple docstring"""


					@slow
					def lowerCamelCase_	(		self							:	Union[str, Any]		):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE				:							Optional[int]						=	RemBertModel.from_pretrained("""google/rembert"""		)
							SCREAMING_SNAKE_CASE				:							Optional[Any]						=	torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]		)
							SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	torch.tensor([[0, 0, 0, 1, 1]]		)
							with torch.no_grad():
									SCREAMING_SNAKE_CASE				:							Optional[Any]						=	model(lowerCamelCase_     ,		token_type_ids=lowerCamelCase_     ,		output_hidden_states=lowerCamelCase_		)

							SCREAMING_SNAKE_CASE				:							List[Any]						=	11_52

							SCREAMING_SNAKE_CASE				:							Optional[int]						=	torch.Size((1, 5, hidden_size)		)
							self.assertEqual(output["""last_hidden_state"""].shape     ,		lowerCamelCase_		)

							SCREAMING_SNAKE_CASE				:							Dict						=	torch.tensor(
							    [
							        [
							            [0.0_754, -0.2_022, 0.1_904],
							            [-0.3_354, -0.3_692, -0.4_791],
							            [-0.2_314, -0.6_729, -0.0_749],
							            [-0.0_396, -0.3_105, -0.4_234],
							            [-0.1_571, -0.0_525, 0.5_353],
							        ]
							    ]		)

							# Running on the original tf implementation gives slightly different results here.
							# Not clear why this variations is present
							# TODO: Find reason for discrepancy
							# expected_original_implementation = [[
							#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
							#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
							#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
							#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
							#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
							# ]]

							self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3]     ,		lowerCamelCase_     ,		atol=1e-4		)		)










'''simple docstring'''




from __future__ import annotations



def 						__A						(       lowerCamelCase_	,	lowerCamelCase_	,	lowerCamelCase_	,	lowerCamelCase_					):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE,    SCREAMING_SNAKE_CASE				:							List[Any]						=	m - 1, 0
		while i >= 0 and j < n:
				if key == mat[i][j]:
						print(f'''Key {key} found at row- {i + 1} column- {j + 1}'''					)
						return
				if key < mat[i][j]:
						i -= 1
				else:
						j += 1
		print(f'''Key {key} not found'''					)



def 						__A						(       ):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE				:							Optional[Any]						=	[[2, 5, 7], [4, 8, 13], [9, 11, 15], [12, 17, 20]]
		SCREAMING_SNAKE_CASE				:							Dict						=	int(input("""Enter the element to be searched:"""					)					)
		print(lowerCamelCase_					)
		search_in_a_sorted_matrix(lowerCamelCase_	,	len(lowerCamelCase_					)	,	len(mat[0]					)	,	lowerCamelCase_					)


if __name__ == "__main__":
							import doctest

							doctest.testmod()
							main()



