

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    RemBertConfig,
			    RemBertForCausalLM,
			    RemBertForMaskedLM,
			    RemBertForMultipleChoice,
			    RemBertForQuestionAnswering,
			    RemBertForSequenceClassification,
			    RemBertForTokenClassification,
			    RemBertModel,
			)
			from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class _UpperCamelCase							:
			'''simple docstring'''







			def __init__(     self							:	List[str] , a							:	int , a							:	Optional[Any]=13 , a							:	Optional[int]=7 , a							:	str=True , a							:	Dict=True , a							:	Tuple=True , a							:	Optional[int]=True , a							:	Optional[Any]=99 , a							:	str=32 , a							:	List[Any]=18 , a							:	Optional[int]=43 , a							:	Optional[Any]=5 , a							:	Any=4 , a							:	Dict=37 , a							:	List[Any]="gelu" , a							:	Any=0.1 , a							:	List[str]=0.1 , a							:	Any=512 , a							:	Optional[Any]=16 , a							:	int=2 , a							:	Union[str, Any]=0.02 , a							:	List[str]=3 , a							:	Dict=4 , a							:	Dict=None , )							->    Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    parent
				SCREAMING_SNAKE_CASE				: Optional[Any]			       =    batch_size
				SCREAMING_SNAKE_CASE				: str			       =    seq_length
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    is_training
				SCREAMING_SNAKE_CASE				: List[Any]			       =    use_input_mask
				SCREAMING_SNAKE_CASE				: int			       =    use_token_type_ids
				SCREAMING_SNAKE_CASE				: List[Any]			       =    use_labels
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    vocab_size
				SCREAMING_SNAKE_CASE				: str			       =    hidden_size
				SCREAMING_SNAKE_CASE				: Dict			       =    input_embedding_size
				SCREAMING_SNAKE_CASE				: List[Any]			       =    output_embedding_size
				SCREAMING_SNAKE_CASE				: int			       =    num_hidden_layers
				SCREAMING_SNAKE_CASE				: Dict			       =    num_attention_heads
				SCREAMING_SNAKE_CASE				: str			       =    intermediate_size
				SCREAMING_SNAKE_CASE				: str			       =    hidden_act
				SCREAMING_SNAKE_CASE				: Dict			       =    hidden_dropout_prob
				SCREAMING_SNAKE_CASE				: int			       =    attention_probs_dropout_prob
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    max_position_embeddings
				SCREAMING_SNAKE_CASE				: Any			       =    type_vocab_size
				SCREAMING_SNAKE_CASE				: Optional[Any]			       =    type_sequence_label_size
				SCREAMING_SNAKE_CASE				: Dict			       =    initializer_range
				SCREAMING_SNAKE_CASE				: List[Any]			       =    num_labels
				SCREAMING_SNAKE_CASE				: Any			       =    num_choices
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    scope
			def 							__UpperCamelCase	(     self							:	List[Any]		)							->    Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    ids_tensor([self.batch_size, self.seq_length] , self.vocab_size		)

				SCREAMING_SNAKE_CASE				: int			       =    None
				if self.use_input_mask:
					SCREAMING_SNAKE_CASE				: int			       =    random_attention_mask([self.batch_size, self.seq_length]		)

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    None
				if self.use_token_type_ids:
					SCREAMING_SNAKE_CASE				: Any			       =    ids_tensor([self.batch_size, self.seq_length] , self.type_vocab_size		)

				SCREAMING_SNAKE_CASE				: Any			       =    None
				SCREAMING_SNAKE_CASE				: Dict			       =    None
				SCREAMING_SNAKE_CASE				: int			       =    None
				if self.use_labels:
					SCREAMING_SNAKE_CASE				: int			       =    ids_tensor([self.batch_size] , self.type_sequence_label_size		)
					SCREAMING_SNAKE_CASE				: Dict			       =    ids_tensor([self.batch_size, self.seq_length] , self.num_labels		)
					SCREAMING_SNAKE_CASE				: Tuple			       =    ids_tensor([self.batch_size] , self.num_choices		)

				SCREAMING_SNAKE_CASE				: str			       =    RemBertConfig(
				    vocab_size=self.vocab_size , hidden_size=self.hidden_size , input_embedding_size=self.input_embedding_size , output_embedding_size=self.output_embedding_size , num_hidden_layers=self.num_hidden_layers , num_attention_heads=self.num_attention_heads , intermediate_size=self.intermediate_size , hidden_act=self.hidden_act , hidden_dropout_prob=self.hidden_dropout_prob , attention_probs_dropout_prob=self.attention_probs_dropout_prob , max_position_embeddings=self.max_position_embeddings , type_vocab_size=self.type_vocab_size , is_decoder=a , initializer_range=self.initializer_range , )

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
			def 							__UpperCamelCase	(     self							:	List[Any]		)							->    Tuple:

				"""simple docstring"""

				(
				    (
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,
				)				: List[Any]			       =    self.prepare_config_and_inputs()

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    True
				SCREAMING_SNAKE_CASE				: Any			       =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
				SCREAMING_SNAKE_CASE				: Tuple			       =    ids_tensor([self.batch_size, self.seq_length] , vocab_size=2		)

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				    encoder_hidden_states,
				    encoder_attention_mask,
				)
			def 							__UpperCamelCase	(     self							:	Optional[int] , a							:	str , a							:	Optional[int] , a							:	Optional[int] , a							:	str , a							:	Union[str, Any] , a							:	Union[str, Any] , a							:	Optional[Any]		)							->    List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[str]			       =    RemBertModel(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: Optional[Any]			       =    model(a , attention_mask=a , token_type_ids=a		)
				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    model(a , token_type_ids=a		)
				SCREAMING_SNAKE_CASE				: Any			       =    model(a		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.hidden_size)		)
			def 							__UpperCamelCase	(     self							:	List[str] , a							:	List[str] , a							:	Optional[int] , a							:	Tuple , a							:	int , a							:	List[str] , a							:	Optional[Any] , a							:	Any , a							:	Tuple , a							:	int , )							->    Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Any			       =    True
				SCREAMING_SNAKE_CASE				: List[str]			       =    RemBertModel(a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: Any			       =    model(
				    a , attention_mask=a , token_type_ids=a , encoder_hidden_states=a , encoder_attention_mask=a , )
				SCREAMING_SNAKE_CASE				: int			       =    model(
				    a , attention_mask=a , token_type_ids=a , encoder_hidden_states=a , )
				SCREAMING_SNAKE_CASE				: int			       =    model(a , attention_mask=a , token_type_ids=a		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.hidden_size)		)
			def 							__UpperCamelCase	(     self							:	List[str] , a							:	Dict , a							:	Optional[Any] , a							:	List[Any] , a							:	Tuple , a							:	List[str] , a							:	Union[str, Any] , a							:	List[str] , a							:	Any , a							:	Any , )							->    Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Optional[Any]			       =    RemBertForCausalLM(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: str			       =    model(a , attention_mask=a , token_type_ids=a , labels=a		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.vocab_size)		)
			def 							__UpperCamelCase	(     self							:	Dict , a							:	Any , a							:	Optional[int] , a							:	Union[str, Any] , a							:	Tuple , a							:	Tuple , a							:	List[str] , a							:	List[str]		)							->    Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Any			       =    RemBertForMaskedLM(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: Dict			       =    model(a , attention_mask=a , token_type_ids=a , labels=a		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.vocab_size)		)
			def 							__UpperCamelCase	(     self							:	Tuple , a							:	str , a							:	str , a							:	Any , a							:	Optional[Any] , a							:	str , a							:	Union[str, Any] , a							:	List[Any] , a							:	Tuple , a							:	Union[str, Any] , )							->    Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: str			       =    True
				SCREAMING_SNAKE_CASE				: str			       =    True
				SCREAMING_SNAKE_CASE				: Any			       =    RemBertForCausalLM(config=a		)
				model.to(a		)
				model.eval()

				# first forward pass
				SCREAMING_SNAKE_CASE				: List[Any]			       =    model(
				    a , attention_mask=a , encoder_hidden_states=a , encoder_attention_mask=a , use_cache=a , )
				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    ids_tensor((self.batch_size, 3) , config.vocab_size		)
				SCREAMING_SNAKE_CASE				: str			       =    ids_tensor((self.batch_size, 3) , vocab_size=2		)

				# append to next input_ids and
				SCREAMING_SNAKE_CASE				: str			       =    torch.cat([input_ids, next_tokens] , dim=-1		)
				SCREAMING_SNAKE_CASE				: Dict			       =    torch.cat([input_mask, next_mask] , dim=-1		)

				SCREAMING_SNAKE_CASE				: List[Any]			       =    model(
				    a , attention_mask=a , encoder_hidden_states=a , encoder_attention_mask=a , output_hidden_states=a , )["hidden_states"][0]
				SCREAMING_SNAKE_CASE				: int			       =    model(
				    a , attention_mask=a , encoder_hidden_states=a , encoder_attention_mask=a , past_key_values=a , output_hidden_states=a , )["hidden_states"][0]

				# select random slice
				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    ids_tensor((1,) , output_from_past.shape[-1]		).item()
				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    output_from_no_past[:, -3:, random_slice_idx].detach()
				SCREAMING_SNAKE_CASE				: Optional[Any]			       =    output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(a , a , atol=1e-3		)		)
			def 							__UpperCamelCase	(     self							:	List[str] , a							:	Optional[Any] , a							:	Dict , a							:	str , a							:	List[Any] , a							:	Any , a							:	Union[str, Any] , a							:	Optional[int]		)							->    List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Any			       =    RemBertForQuestionAnswering(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: str			       =    model(
				    a , attention_mask=a , token_type_ids=a , start_positions=a , end_positions=a , )
				self.parent.assertEqual(result.start_logits.shape , (self.batch_size, self.seq_length)		)
				self.parent.assertEqual(result.end_logits.shape , (self.batch_size, self.seq_length)		)
			def 							__UpperCamelCase	(     self							:	Optional[Any] , a							:	str , a							:	List[str] , a							:	int , a							:	int , a							:	Union[str, Any] , a							:	List[Any] , a							:	str		)							->    Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Tuple			       =    self.num_labels
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    RemBertForSequenceClassification(a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: Any			       =    model(a , attention_mask=a , token_type_ids=a , labels=a		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.num_labels)		)
			def 							__UpperCamelCase	(     self							:	int , a							:	List[str] , a							:	Dict , a							:	Optional[Any] , a							:	Optional[Any] , a							:	Tuple , a							:	Tuple , a							:	Union[str, Any]		)							->    str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[Any]			       =    self.num_labels
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    RemBertForTokenClassification(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    model(a , attention_mask=a , token_type_ids=a , labels=a		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.num_labels)		)
			def 							__UpperCamelCase	(     self							:	Union[str, Any] , a							:	Dict , a							:	Dict , a							:	Any , a							:	Optional[Any] , a							:	Tuple , a							:	Tuple , a							:	str		)							->    str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[str]			       =    self.num_choices
				SCREAMING_SNAKE_CASE				: Any			       =    RemBertForMultipleChoice(config=a		)
				model.to(a		)
				model.eval()
				SCREAMING_SNAKE_CASE				: Optional[int]			       =    input_ids.unsqueeze(1		).expand(-1 , self.num_choices , -1		).contiguous()
				SCREAMING_SNAKE_CASE				: str			       =    token_type_ids.unsqueeze(1		).expand(-1 , self.num_choices , -1		).contiguous()
				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    input_mask.unsqueeze(1		).expand(-1 , self.num_choices , -1		).contiguous()
				SCREAMING_SNAKE_CASE				: Any			       =    model(
				    a , attention_mask=a , token_type_ids=a , labels=a , )
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.num_choices)		)






			def 							__UpperCamelCase	(     self							:	List[str]		)							->    Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[str]			       =    self.prepare_config_and_inputs()
				(
				    (
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,
				)				: List[str]			       =    config_and_inputs
				SCREAMING_SNAKE_CASE				: Any			       =    {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
				return config, inputs_dict






@require_torch
class _UpperCamelCase							(							__A							,		__A							,		unittest.TestCase				):
			'''simple docstring'''







			lowerCamelCase__          =(
			    (
			        RemBertModel,
			        RemBertForMaskedLM,
			        RemBertForCausalLM,
			        RemBertForMultipleChoice,
			        RemBertForQuestionAnswering,
			        RemBertForSequenceClassification,
			        RemBertForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCamelCase__          =(RemBertForCausalLM,) if is_torch_available() else ()
			lowerCamelCase__          =(
			    {
			        'feature-extraction': RemBertModel,
			        'fill-mask': RemBertForMaskedLM,
			        'question-answering': RemBertForQuestionAnswering,
			        'text-classification': RemBertForSequenceClassification,
			        'text-generation': RemBertForCausalLM,
			        'token-classification': RemBertForTokenClassification,
			        'zero-shot': RemBertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			def 							__UpperCamelCase	(     self							:	Union[str, Any]		)							->    Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Tuple			       =    RemBertModelTester(self		)
				SCREAMING_SNAKE_CASE				: Any			       =    ConfigTester(self , config_class=a , hidden_size=37		)
			def 							__UpperCamelCase	(     self							:	str		)							->    List[Any]:

				"""simple docstring"""

				self.config_tester.run_common_tests()
			def 							__UpperCamelCase	(     self							:	Tuple		)							->    Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Tuple			       =    self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*a		)
			def 							__UpperCamelCase	(     self							:	str		)							->    str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Optional[int]			       =    self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
					SCREAMING_SNAKE_CASE				: Any			       =    type
					self.model_tester.create_and_check_model(*a		)
			def 							__UpperCamelCase	(     self							:	List[Any]		)							->    List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[Any]			       =    self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*a		)
			def 							__UpperCamelCase	(     self							:	Union[str, Any]		)							->    Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*a		)
			def 							__UpperCamelCase	(     self							:	Optional[Any]		)							->    str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: int			       =    self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_decoder_model_past_large_inputs(*a		)
			def 							__UpperCamelCase	(     self							:	List[Any]		)							->    List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: str			       =    self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*a		)
			def 							__UpperCamelCase	(     self							:	Dict		)							->    int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Optional[int]			       =    self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*a		)
			def 							__UpperCamelCase	(     self							:	Tuple		)							->    List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[str]			       =    self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_token_classification(*a		)
			def 							__UpperCamelCase	(     self							:	Union[str, Any]		)							->    Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Optional[Any]			       =    self.model_tester.prepare_config_and_inputs_for_decoder()
				self.model_tester.create_and_check_model_as_decoder(*a		)
			def 							__UpperCamelCase	(     self							:	Any		)							->    Dict:

				"""simple docstring"""

				(
				    (
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,(
				    SCREAMING_SNAKE_CASE
				)       ,
				)				: int			       =    self.model_tester.prepare_config_and_inputs_for_decoder()

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    None

				self.model_tester.create_and_check_model_as_decoder(
				    a , a , a , a , a , a , a , a , a , )






			@slow
			def 							__UpperCamelCase	(     self							:	List[Any]		)							->    Optional[Any]:

				"""simple docstring"""

				for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					SCREAMING_SNAKE_CASE				: List[Any]			       =    RemBertModel.from_pretrained(a		)
					self.assertIsNotNone(a		)






@require_torch
class _UpperCamelCase							(							unittest.TestCase				):
			'''simple docstring'''







			@slow
			def 							__UpperCamelCase	(     self							:	int		)							->    Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: List[str]			       =    RemBertModel.from_pretrained("google/rembert"		)
				SCREAMING_SNAKE_CASE				: Any			       =    torch.tensor([[312, 5_6498, 313, 2125, 313]]		)
				SCREAMING_SNAKE_CASE				: Tuple			       =    torch.tensor([[0, 0, 0, 1, 1]]		)
				with torch.no_grad():
					SCREAMING_SNAKE_CASE				: Tuple			       =    model(a , token_type_ids=a , output_hidden_states=a		)

				SCREAMING_SNAKE_CASE				: Tuple			       =    1152

				SCREAMING_SNAKE_CASE				: List[Any]			       =    torch.Size((1, 5, hidden_size)		)
				self.assertEqual(output["last_hidden_state"].shape , a		)

				SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    torch.tensor(
				    [
				        [
				            [0.0754, -0.2022, 0.1904],
				            [-0.3354, -0.3692, -0.4791],
				            [-0.2314, -0.6729, -0.0749],
				            [-0.0396, -0.3105, -0.4234],
				            [-0.1571, -0.0525, 0.5353],
				        ]
				    ]		)

				# Running on the original tf implementation gives slightly different results here.
				# Not clear why this variations is present
				# TODO: Find reason for discrepancy
				# expected_original_implementation = [[
				#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
				#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
				#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
				#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
				#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
				# ]]

				self.assertTrue(torch.allclose(output["last_hidden_state"][:, :, :3] , a , atol=1e-4		)		)


from __future__ import annotations

from typing import Any






class _UpperCamelCase							:
			'''simple docstring'''







			def __init__(     self							:	Dict , a							:	Any , a							:	Any		)							->    None:  # noqa: A002

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: str			       =    item
				SCREAMING_SNAKE_CASE				: List[Any]			       =    next






class _UpperCamelCase							:
			'''simple docstring'''







			def __init__(     self							:	Optional[int]		)							->    None:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Node | None			       =    None
				SCREAMING_SNAKE_CASE				: List[Any]			       =    0
			def 							__UpperCamelCase	(     self							:	Union[str, Any] , a							:	Any		)							->    None:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE				: Any			       =    Node(a , self.head		)
				self.size += 1
			def 							__UpperCamelCase	(     self							:	Dict		)							->    Any:

				"""simple docstring"""

				if self.head is None:
					return None
				else:
					SCREAMING_SNAKE_CASE				: Optional[int]			       =    self.head.item
					SCREAMING_SNAKE_CASE				: Dict			       =    self.head.next
					self.size -= 1
					return item
			def 							__UpperCamelCase	(     self							:	Any		)							->    bool:

				"""simple docstring"""

				return self.head is None
			def __str__(     self							:	Union[str, Any]		)							->    str:

				"""simple docstring"""

				if self.is_empty():
					return ""
				else:
					SCREAMING_SNAKE_CASE				: Union[str, Any]			       =    self.head
					SCREAMING_SNAKE_CASE				: str			       =    ""
					SCREAMING_SNAKE_CASE				: list[str]			       =    []
					while iterate:
						item_list.append(str(iterate.item		)		)
						SCREAMING_SNAKE_CASE				: str			       =    iterate.next

					SCREAMING_SNAKE_CASE				: List[str]			       =    " --> ".join(a		)

					return item_str






			def __len__(     self							:	List[Any]		)							->    int:

				"""simple docstring"""

				return self.size