






import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__     :      Optional[int]             =			logging.get_logger(__name__)

A__     :      Optional[int]             =			{
    'source_spm': 'source.spm',
    'target_spm': 'target.spm',
    'vocab': 'vocab.json',
    'target_vocab_file': 'target_vocab.json',
    'tokenizer_config_file': 'tokenizer_config.json',
}

A__     :      Optional[Any]             =			{
    'source_spm': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm'
    },
    'target_spm': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm'
    },
    'vocab': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json'
    },
    'tokenizer_config_file': {
        'Helsinki-NLP/opus-mt-en-de': (
            'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json'
        )
    },
}

A__     :      int             =			{'Helsinki-NLP/opus-mt-en-de': 5_12}
A__     :      List[str]             =			{}
class 				_UpperCAmelCase (	_UpperCAmelCase     ):

				"""simple docstring"""




				lowercase__    							=	VOCAB_FILES_NAMES
				lowercase__    							=	PRETRAINED_VOCAB_FILES_MAP
				lowercase__    							=	PRETRAINED_INIT_CONFIGURATION
				lowercase__    							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__    							=	["""input_ids""", """attention_mask"""]
				lowercase__    							=	re.compile(""">>.+<<"""     )  # type: re.Pattern




				def __init__(					self						:   Union[str, Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Dict,  lowerCamelCase						:   int=None,  lowerCamelCase						:   Dict=None,  lowerCamelCase						:   Optional[Any]=None,  lowerCamelCase						:   str="<unk>",  lowerCamelCase						:   Tuple="</s>",  lowerCamelCase						:   Union[str, Any]="<pad>",  lowerCamelCase						:   Optional[Any]=512,  lowerCamelCase						:   Union[str, Any] = None,  lowerCamelCase						:   List[Any]=False,  **lowerCamelCase						:   List[Any],  ):

								'''simple docstring'''
								lowercase__       			=      {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
								    source_lang=lowercase__,  target_lang=lowercase__,  unk_token=lowercase__,  eos_token=lowercase__,  pad_token=lowercase__,  model_max_length=lowercase__,  sp_model_kwargs=self.sp_model_kwargs,  target_vocab_file=lowercase__,  separate_vocabs=lowercase__,  **lowercase__,  )
								assert Path(lowercase__  ).exists(), F"""cannot find spm source {source_spm}"""

								lowercase__       			=      separate_vocabs
								lowercase__       			=      load_json(lowercase__  )
								if self.unk_token not in self.encoder:
												raise KeyError('''<unk> token must be in vocab'''  )
								assert self.pad_token in self.encoder

								if separate_vocabs:
												lowercase__       			=      load_json(lowercase__  )
												lowercase__       			=      {v: k for k, v in self.target_encoder.items()}
												lowercase__       			=      []
								else:
												lowercase__       			=      {v: k for k, v in self.encoder.items()}
												lowercase__       			=      [k for k in self.encoder if k.startswith('''>>'''  ) and k.endswith('''<<'''  )]

								lowercase__       			=      source_lang
								lowercase__       			=      target_lang
								lowercase__       			=      [source_spm, target_spm]

								# load SentencePiece model for pre-processing
								lowercase__       			=      load_spm(lowercase__,  self.sp_model_kwargs  )
								lowercase__       			=      load_spm(lowercase__,  self.sp_model_kwargs  )
								lowercase__       			=      self.spm_source
								lowercase__       			=      self.encoder

								# Multilingual target side: default to using first supported language code.

								self._setup_normalizer()




				def     lowercase__       (					self						:   Any  ):

								'''simple docstring'''






								try:
												from sacremoses import MosesPunctNormalizer

												lowercase__       			=      MosesPunctNormalizer(self.source_lang  ).normalize
								except (ImportError, FileNotFoundError):
												warnings.warn('''Recommended: pip install sacremoses.'''  )
												lowercase__       			=      lambda lowerCamelCase  : x




				def     lowercase__       (					self						:   Union[str, Any],  lowerCamelCase						:   Tuple  ):

								'''simple docstring'''
								return self.punc_normalizer(lowercase__  ) if x else ""




				def     lowercase__       (					self						:   Optional[Any],  lowerCamelCase						:   Tuple  ):

								'''simple docstring'''
								return self.current_encoder.get(lowercase__,  self.current_encoder[self.unk_token]  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Tuple  ):

								'''simple docstring'''
								lowercase__       			=      self.language_code_re.match(lowercase__  )
								lowercase__       			=      [match.group(0  )] if match else []
								return code, self.language_code_re.sub('''''',  lowercase__  )




				def     lowercase__       (					self						:   int,  lowerCamelCase						:   str  ):

								'''simple docstring'''
								lowercase__       			=      self.remove_language_code(lowercase__  )
								lowercase__       			=      self.current_spm.encode(lowercase__,  out_type=lowercase__  )
								return code + pieces




				def     lowercase__       (					self						:   Union[str, Any],  lowerCamelCase						:   Optional[Any]  ):

								'''simple docstring'''
								return self.decoder.get(lowercase__,  self.unk_token  )




				def     lowercase__       (					self						:   Tuple,  lowerCamelCase						:   Dict,  **lowerCamelCase						:   List[str]  ):

								'''simple docstring'''
								return super().batch_decode(lowercase__,  **lowercase__  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Union[str, Any],  **lowerCamelCase						:   List[str]  ):

								'''simple docstring'''
								return super().decode(lowercase__,  **lowercase__  )




				def     lowercase__       (					self						:   List[Any],  lowerCamelCase						:   Dict  ):

								'''simple docstring'''
								lowercase__       			=      self.spm_source if self._decode_use_source_tokenizer else self.spm_target
								lowercase__       			=      []
								lowercase__       			=      ""
								for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																out_string += sp_model.decode_pieces(lowercase__  ) + token + " "
																lowercase__       			=      []
												else:
																current_sub_tokens.append(lowercase__  )
								out_string += sp_model.decode_pieces(lowercase__  )
								return out_string.strip()




				def     lowercase__       (					self						:   Union[str, Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Dict=None  ):

								'''simple docstring'''






								if token_ids_a is None:
												return token_ids_a + [self.eos_token_id]
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return token_ids_a + token_ids_a + [self.eos_token_id]




				def     lowercase__       (					self						:   Any  ):

								'''simple docstring'''
								lowercase__       			=      self.spm_source
								lowercase__       			=      self.encoder




				def     lowercase__       (					self						:   Dict  ):

								'''simple docstring'''
								lowercase__       			=      self.spm_target
								if self.separate_vocabs:
												lowercase__       			=      self.target_encoder




				@property
				def     lowercase__       (					self						:   Dict  ):

								'''simple docstring'''
								return len(self.encoder  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   str,  lowerCamelCase						:   int = None  ):

								'''simple docstring'''






								if not os.path.isdir(lowercase__  ):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
												return
								lowercase__       			=      []

								if self.separate_vocabs:
												lowercase__       			=      os.path.join(
												    lowercase__,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab'''],  )
												lowercase__       			=      os.path.join(
												    lowercase__,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''target_vocab_file'''],  )
												save_json(self.encoder,  lowercase__  )
												save_json(self.target_encoder,  lowercase__  )
												saved_files.append(lowercase__  )
												saved_files.append(lowercase__  )
								else:
												lowercase__       			=      os.path.join(
												    lowercase__,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab''']  )
												save_json(self.encoder,  lowercase__  )
												saved_files.append(lowercase__  )

								for spm_save_filename, spm_orig_path, spm_model in zip(
								    [VOCAB_FILES_NAMES['''source_spm'''], VOCAB_FILES_NAMES['''target_spm''']],  self.spm_files,  [self.spm_source, self.spm_target],  ):
												lowercase__       			=      os.path.join(
												    lowercase__,  (filename_prefix + '''-''' if filename_prefix else '''''') + spm_save_filename  )
												if os.path.abspath(lowercase__  ) != os.path.abspath(lowercase__  ) and os.path.isfile(lowercase__  ):
																copyfile(lowercase__,  lowercase__  )
																saved_files.append(lowercase__  )
												elif not os.path.isfile(lowercase__  ):
																with open(lowercase__,  '''wb'''  ) as fi:
																				lowercase__       			=      spm_model.serialized_model_proto()
																				fi.write(lowercase__  )
																saved_files.append(lowercase__  )

								return tuple(lowercase__  )




				def     lowercase__       (					self						:   str  ):

								'''simple docstring'''
								return self.get_src_vocab()




				def     lowercase__       (					self						:   Optional[int]  ):

								'''simple docstring'''
								return dict(self.encoder,  **self.added_tokens_encoder  )




				def     lowercase__       (					self						:   Tuple  ):

								'''simple docstring'''
								return dict(self.target_encoder,  **self.added_tokens_decoder  )




				def __getstate__(					self						:   str  ):

								'''simple docstring'''
								lowercase__       			=      self.__dict__.copy()
								state.update(
								    {k: None for k in ['''spm_source''', '''spm_target''', '''current_spm''', '''punc_normalizer''', '''target_vocab_file''']}  )
								return state




				def __setstate__(					self						:   Optional[Any],  lowerCamelCase						:   List[Any]  ):

								'''simple docstring'''
								lowercase__       			=      d

								# for backward compatibility
								if not hasattr(self,  '''sp_model_kwargs'''  ):
												lowercase__       			=      {}

								lowercase__       			=      (load_spm(lowercase__,  self.sp_model_kwargs  ) for f in self.spm_files)
								lowercase__       			=      self.spm_source
								self._setup_normalizer()




				def     lowercase__       (					self						:   int,  *lowerCamelCase						:   List[str],  **lowerCamelCase						:   List[Any]  ):

								'''simple docstring'''
								return 1




				def     lowercase__       (					self						:   Optional[int],  lowerCamelCase						:   Optional[Any]  ):

								'''simple docstring'''
								lowercase__       			=      set(self.all_special_ids  )  # call it once instead of inside list comp
								all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special
								return [1 if x in all_special_ids else 0 for x in seq]




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Union[str, Any] = None,  lowerCamelCase						:   Union[str, Any] = False  ):

								'''simple docstring'''






								if already_has_special_tokens:
												return self._special_token_mask(lowercase__  )
								elif token_ids_a is None:
												return self._special_token_mask(lowercase__  ) + [1]
								else:
												return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]






def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				lowercase__       			=      sentencepiece.SentencePieceProcessor(**SCREAMING_SNAKE_CASE_    )
				spm.Load(SCREAMING_SNAKE_CASE_    )
				return spm






def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''

				with open(SCREAMING_SNAKE_CASE_		,       '''w'''    ) as f:
								json.dump(SCREAMING_SNAKE_CASE_		,       SCREAMING_SNAKE_CASE_		,       indent=2    )






def   a      (						lowerCamelCase_    ):






				'''simple docstring'''

				with open(SCREAMING_SNAKE_CASE_		,       '''r'''    ) as f:
								return json.load(SCREAMING_SNAKE_CASE_    )












import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RemBertConfig,
							    RemBertForCausalLM,
							    RemBertForMaskedLM,
							    RemBertForMultipleChoice,
							    RemBertForQuestionAnswering,
							    RemBertForSequenceClassification,
							    RemBertForTokenClassification,
							    RemBertModel,
							)
							from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class 				_UpperCAmelCase :

				"""simple docstring"""




				def __init__(					self						:   Optional[int],  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   int=13,  lowerCamelCase						:   str=7,  lowerCamelCase						:   int=True,  lowerCamelCase						:   Optional[Any]=True,  lowerCamelCase						:   List[str]=True,  lowerCamelCase						:   str=True,  lowerCamelCase						:   List[str]=99,  lowerCamelCase						:   Any=32,  lowerCamelCase						:   int=18,  lowerCamelCase						:   Optional[int]=43,  lowerCamelCase						:   Optional[Any]=5,  lowerCamelCase						:   int=4,  lowerCamelCase						:   Union[str, Any]=37,  lowerCamelCase						:   List[str]="gelu",  lowerCamelCase						:   List[str]=0.1,  lowerCamelCase						:   Dict=0.1,  lowerCamelCase						:   Dict=512,  lowerCamelCase						:   str=16,  lowerCamelCase						:   Optional[Any]=2,  lowerCamelCase						:   int=0.02,  lowerCamelCase						:   Optional[int]=3,  lowerCamelCase						:   Optional[int]=4,  lowerCamelCase						:   Union[str, Any]=None,  ):

								'''simple docstring'''
								lowercase__       			=      parent
								lowercase__       			=      batch_size
								lowercase__       			=      seq_length
								lowercase__       			=      is_training
								lowercase__       			=      use_input_mask
								lowercase__       			=      use_token_type_ids
								lowercase__       			=      use_labels
								lowercase__       			=      vocab_size
								lowercase__       			=      hidden_size
								lowercase__       			=      input_embedding_size
								lowercase__       			=      output_embedding_size
								lowercase__       			=      num_hidden_layers
								lowercase__       			=      num_attention_heads
								lowercase__       			=      intermediate_size
								lowercase__       			=      hidden_act
								lowercase__       			=      hidden_dropout_prob
								lowercase__       			=      attention_probs_dropout_prob
								lowercase__       			=      max_position_embeddings
								lowercase__       			=      type_vocab_size
								lowercase__       			=      type_sequence_label_size
								lowercase__       			=      initializer_range
								lowercase__       			=      num_labels
								lowercase__       			=      num_choices
								lowercase__       			=      scope




				def     lowercase__       (					self						:   Optional[Any]  ):

								'''simple docstring'''
								lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  self.vocab_size  )

								lowercase__       			=      None
								if self.use_input_mask:
												lowercase__       			=      random_attention_mask([self.batch_size, self.seq_length]  )

								lowercase__       			=      None
								if self.use_token_type_ids:
												lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  self.type_vocab_size  )

								lowercase__       			=      None
								lowercase__       			=      None
								lowercase__       			=      None
								if self.use_labels:
												lowercase__       			=      ids_tensor([self.batch_size],  self.type_sequence_label_size  )
												lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  self.num_labels  )
												lowercase__       			=      ids_tensor([self.batch_size],  self.num_choices  )

								lowercase__       			=      RemBertConfig(
								    vocab_size=self.vocab_size,  hidden_size=self.hidden_size,  input_embedding_size=self.input_embedding_size,  output_embedding_size=self.output_embedding_size,  num_hidden_layers=self.num_hidden_layers,  num_attention_heads=self.num_attention_heads,  intermediate_size=self.intermediate_size,  hidden_act=self.hidden_act,  hidden_dropout_prob=self.hidden_dropout_prob,  attention_probs_dropout_prob=self.attention_probs_dropout_prob,  max_position_embeddings=self.max_position_embeddings,  type_vocab_size=self.type_vocab_size,  is_decoder=lowerCamelCase,  initializer_range=self.initializer_range,  )

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def     lowercase__       (					self						:   Union[str, Any]  ):

								'''simple docstring'''
								(
								    (
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						
								)       			=      self.prepare_config_and_inputs()

								lowercase__       			=      True
								lowercase__       			=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
								lowercase__       			=      ids_tensor([self.batch_size, self.seq_length],  vocab_size=2  )

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								    encoder_hidden_states,
								    encoder_attention_mask,
								)




				def     lowercase__       (					self						:   Tuple,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   List[str],  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   int,  lowerCamelCase						:   List[Any],  lowerCamelCase						:   str  ):

								'''simple docstring'''
								lowercase__       			=      RemBertModel(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase  )
								lowercase__       			=      model(lowerCamelCase,  token_type_ids=lowerCamelCase  )
								lowercase__       			=      model(lowerCamelCase  )
								self.parent.assertEqual(result.last_hidden_state.shape,  (self.batch_size, self.seq_length, self.hidden_size)  )




				def     lowercase__       (					self						:   Optional[Any],  lowerCamelCase						:   List[str],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Dict,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   str,  lowerCamelCase						:   Any,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Any,  ):

								'''simple docstring'''
								lowercase__       			=      True
								lowercase__       			=      RemBertModel(lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  )
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  )
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase  )
								self.parent.assertEqual(result.last_hidden_state.shape,  (self.batch_size, self.seq_length, self.hidden_size)  )




				def     lowercase__       (					self						:   str,  lowerCamelCase						:   str,  lowerCamelCase						:   str,  lowerCamelCase						:   int,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   Optional[Any],  ):

								'''simple docstring'''
								lowercase__       			=      RemBertForCausalLM(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.seq_length, self.vocab_size)  )




				def     lowercase__       (					self						:   Any,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Dict,  lowerCamelCase						:   str,  lowerCamelCase						:   List[str],  lowerCamelCase						:   Any,  lowerCamelCase						:   List[str]  ):

								'''simple docstring'''
								lowercase__       			=      RemBertForMaskedLM(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.seq_length, self.vocab_size)  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   List[str],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Any,  lowerCamelCase						:   Union[str, Any],  lowerCamelCase						:   int,  lowerCamelCase						:   Dict,  ):

								'''simple docstring'''
								lowercase__       			=      True
								lowercase__       			=      True
								lowercase__       			=      RemBertForCausalLM(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()

								# first forward pass
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  use_cache=lowerCamelCase,  )
								lowercase__       			=      outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								lowercase__       			=      ids_tensor((self.batch_size, 3),  config.vocab_size  )
								lowercase__       			=      ids_tensor((self.batch_size, 3),  vocab_size=2  )

								# append to next input_ids and
								lowercase__       			=      torch.cat([input_ids, next_tokens],  dim=-1  )
								lowercase__       			=      torch.cat([input_mask, next_mask],  dim=-1  )

								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  output_hidden_states=lowerCamelCase,  )['''hidden_states'''][0]
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  encoder_attention_mask=lowerCamelCase,  past_key_values=lowerCamelCase,  output_hidden_states=lowerCamelCase,  )['''hidden_states'''][0]

								# select random slice
								lowercase__       			=      ids_tensor((1,),  output_from_past.shape[-1]  ).item()
								lowercase__       			=      output_from_no_past[:, -3:, random_slice_idx].detach()
								lowercase__       			=      output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(lowerCamelCase,  lowerCamelCase,  atol=1E-3  )  )




				def     lowercase__       (					self						:   Optional[int],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   List[Any],  lowerCamelCase						:   List[str],  lowerCamelCase						:   List[Any]  ):

								'''simple docstring'''
								lowercase__       			=      RemBertForQuestionAnswering(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  start_positions=lowerCamelCase,  end_positions=lowerCamelCase,  )
								self.parent.assertEqual(result.start_logits.shape,  (self.batch_size, self.seq_length)  )
								self.parent.assertEqual(result.end_logits.shape,  (self.batch_size, self.seq_length)  )




				def     lowercase__       (					self						:   List[str],  lowerCamelCase						:   Dict,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   List[Any],  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Union[str, Any]  ):

								'''simple docstring'''
								lowercase__       			=      self.num_labels
								lowercase__       			=      RemBertForSequenceClassification(lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.num_labels)  )




				def     lowercase__       (					self						:   Tuple,  lowerCamelCase						:   Optional[int],  lowerCamelCase						:   Dict,  lowerCamelCase						:   str,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   Tuple,  lowerCamelCase						:   Optional[int]  ):

								'''simple docstring'''
								lowercase__       			=      self.num_labels
								lowercase__       			=      RemBertForTokenClassification(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      model(lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.seq_length, self.num_labels)  )




				def     lowercase__       (					self						:   Optional[int],  lowerCamelCase						:   Any,  lowerCamelCase						:   Tuple,  lowerCamelCase						:   int,  lowerCamelCase						:   str,  lowerCamelCase						:   int,  lowerCamelCase						:   Optional[Any],  lowerCamelCase						:   Any  ):

								'''simple docstring'''
								lowercase__       			=      self.num_choices
								lowercase__       			=      RemBertForMultipleChoice(config=lowerCamelCase  )
								model.to(lowerCamelCase  )
								model.eval()
								lowercase__       			=      input_ids.unsqueeze(1  ).expand(-1,  self.num_choices,  -1  ).contiguous()
								lowercase__       			=      token_type_ids.unsqueeze(1  ).expand(-1,  self.num_choices,  -1  ).contiguous()
								lowercase__       			=      input_mask.unsqueeze(1  ).expand(-1,  self.num_choices,  -1  ).contiguous()
								lowercase__       			=      model(
								    lowerCamelCase,  attention_mask=lowerCamelCase,  token_type_ids=lowerCamelCase,  labels=lowerCamelCase,  )
								self.parent.assertEqual(result.logits.shape,  (self.batch_size, self.num_choices)  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								lowercase__       			=      self.prepare_config_and_inputs()
								(
								    (
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						
								)       			=      config_and_inputs
								lowercase__       			=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
								return config, inputs_dict



@require_torch
class 				_UpperCAmelCase (	A__		,A__		,unittest.TestCase     ):

				"""simple docstring"""




				lowercase__    							=	(
				    (
				        RemBertModel,
				        RemBertForMaskedLM,
				        RemBertForCausalLM,
				        RemBertForMultipleChoice,
				        RemBertForQuestionAnswering,
				        RemBertForSequenceClassification,
				        RemBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				lowercase__    							=	(RemBertForCausalLM,) if is_torch_available() else ()
				lowercase__    							=	(
				    {
				        """feature-extraction""": RemBertModel,
				        """fill-mask""": RemBertForMaskedLM,
				        """question-answering""": RemBertForQuestionAnswering,
				        """text-classification""": RemBertForSequenceClassification,
				        """text-generation""": RemBertForCausalLM,
				        """token-classification""": RemBertForTokenClassification,
				        """zero-shot""": RemBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)




				def     lowercase__       (					self						:   str  ):

								'''simple docstring'''
								lowercase__       			=      RemBertModelTester(self  )
								lowercase__       			=      ConfigTester(self,  config_class=lowerCamelCase,  hidden_size=37  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								self.config_tester.run_common_tests()




				def     lowercase__       (					self						:   List[Any]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCamelCase  )




				def     lowercase__       (					self						:   Dict  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
												lowercase__       			=      type
												self.model_tester.create_and_check_model(*lowerCamelCase  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase  )




				def     lowercase__       (					self						:   Optional[int]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase  )




				def     lowercase__       (					self						:   Dict  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCamelCase  )




				def     lowercase__       (					self						:   Tuple  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*lowerCamelCase  )




				def     lowercase__       (					self						:   Union[str, Any]  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase  )




				def     lowercase__       (					self						:   Tuple  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*lowerCamelCase  )




				def     lowercase__       (					self						:   Any  ):

								'''simple docstring'''
								lowercase__       			=      self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase  )




				def     lowercase__       (					self						:   List[str]  ):

								'''simple docstring'''
								# This regression test was failing with PyTorch < 1.3
								(
								    (
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						(
								    lowercase__
								)			,						
								)       			=      self.model_tester.prepare_config_and_inputs_for_decoder()

								lowercase__       			=      None

								self.model_tester.create_and_check_model_as_decoder(
								    lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  )




				@slow
				def     lowercase__       (					self						:   int  ):

								'''simple docstring'''
								for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												lowercase__       			=      RemBertModel.from_pretrained(lowerCamelCase  )
												self.assertIsNotNone(lowerCamelCase  )



@require_torch
class 				_UpperCAmelCase (	unittest.TestCase     ):

				"""simple docstring"""




				@slow
				def     lowercase__       (					self						:   int  ):

								'''simple docstring'''
								# Test exact values at the last hidden layer
								lowercase__       			=      RemBertModel.from_pretrained('''google/rembert'''  )
								lowercase__       			=      torch.tensor([[312, 56_498, 313, 2_125, 313]]  )
								lowercase__       			=      torch.tensor([[0, 0, 0, 1, 1]]  )
								with torch.no_grad():
												lowercase__       			=      model(lowerCamelCase,  token_type_ids=lowerCamelCase,  output_hidden_states=lowerCamelCase  )

								lowercase__       			=      1_152

								lowercase__       			=      torch.Size((1, 5, hidden_size)  )
								self.assertEqual(output['''last_hidden_state'''].shape,  lowerCamelCase  )

								lowercase__       			=      torch.tensor(
								    [
								        [
								            [0.0754, -0.2022, 0.1904],
								            [-0.3354, -0.3692, -0.4791],
								            [-0.2314, -0.6729, -0.0749],
								            [-0.0396, -0.3105, -0.4234],
								            [-0.1571, -0.0525, 0.5353],
								        ]
								    ]  )

								# Running on the original tf implementation gives slightly different results here.
								# Not clear why this variations is present
								# TODO: Find reason for discrepancy
								# expected_original_implementation = [[
								#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
								#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
								#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
								#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
								#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
								# ]]

								self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3],  lowerCamelCase,  atol=1E-4  )  )




