




'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       RemBertConfig,
       RemBertForCausalLM,
       RemBertForMaskedLM,
       RemBertForMultipleChoice,
       RemBertForQuestionAnswering,
       RemBertForSequenceClassification,
       RemBertForTokenClassification,
       RemBertModel,
   )
   from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class       _a  :
       def __init__(     self						:     Tuple       ,    lowercase						:     List[str]       ,    lowercase						:     Optional[int]=13       ,    lowercase						:     Union[str, Any]=7       ,    lowercase						:     List[Any]=True       ,    lowercase						:     int=True       ,    lowercase						:     Dict=True       ,    lowercase						:     Optional[int]=True       ,    lowercase						:     Tuple=99       ,    lowercase						:     Dict=32       ,    lowercase						:     Tuple=18       ,    lowercase						:     List[Any]=43       ,    lowercase						:     Union[str, Any]=5       ,    lowercase						:     List[str]=4       ,    lowercase						:     Union[str, Any]=37       ,    lowercase						:     str="gelu"       ,    lowercase						:     Dict=0.1       ,    lowercase						:     Union[str, Any]=0.1       ,    lowercase						:     Dict=512       ,    lowercase						:     Tuple=16       ,    lowercase						:     int=2       ,    lowercase						:     Any=0.02       ,    lowercase						:     Union[str, Any]=3       ,    lowercase						:     Tuple=4       ,    lowercase						:     Tuple=None       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		parent
           UpperCAmelCase      	=		batch_size
           UpperCAmelCase      	=		seq_length
           UpperCAmelCase      	=		is_training
           UpperCAmelCase      	=		use_input_mask
           UpperCAmelCase      	=		use_token_type_ids
           UpperCAmelCase      	=		use_labels
           UpperCAmelCase      	=		vocab_size
           UpperCAmelCase      	=		hidden_size
           UpperCAmelCase      	=		input_embedding_size
           UpperCAmelCase      	=		output_embedding_size
           UpperCAmelCase      	=		num_hidden_layers
           UpperCAmelCase      	=		num_attention_heads
           UpperCAmelCase      	=		intermediate_size
           UpperCAmelCase      	=		hidden_act
           UpperCAmelCase      	=		hidden_dropout_prob
           UpperCAmelCase      	=		attention_probs_dropout_prob
           UpperCAmelCase      	=		max_position_embeddings
           UpperCAmelCase      	=		type_vocab_size
           UpperCAmelCase      	=		type_sequence_label_size
           UpperCAmelCase      	=		initializer_range
           UpperCAmelCase      	=		num_labels
           UpperCAmelCase      	=		num_choices
           UpperCAmelCase      	=		scope
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		ids_tensor([self.batch_size, self.seq_length]       ,    self.vocab_size    )

           UpperCAmelCase      	=		None
           if self.use_input_mask:
               UpperCAmelCase      	=		random_attention_mask([self.batch_size, self.seq_length]    )

           UpperCAmelCase      	=		None
           if self.use_token_type_ids:
               UpperCAmelCase      	=		ids_tensor([self.batch_size, self.seq_length]       ,    self.type_vocab_size    )

           UpperCAmelCase      	=		None
           UpperCAmelCase      	=		None
           UpperCAmelCase      	=		None
           if self.use_labels:
               UpperCAmelCase      	=		ids_tensor([self.batch_size]       ,    self.type_sequence_label_size    )
               UpperCAmelCase      	=		ids_tensor([self.batch_size, self.seq_length]       ,    self.num_labels    )
               UpperCAmelCase      	=		ids_tensor([self.batch_size]       ,    self.num_choices    )

           UpperCAmelCase      	=		RemBertConfig(
               vocab_size=self.vocab_size       ,    hidden_size=self.hidden_size       ,    input_embedding_size=self.input_embedding_size       ,    output_embedding_size=self.output_embedding_size       ,    num_hidden_layers=self.num_hidden_layers       ,    num_attention_heads=self.num_attention_heads       ,    intermediate_size=self.intermediate_size       ,    hidden_act=self.hidden_act       ,    hidden_dropout_prob=self.hidden_dropout_prob       ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,    max_position_embeddings=self.max_position_embeddings       ,    type_vocab_size=self.type_vocab_size       ,    is_decoder=lowercase       ,    initializer_range=self.initializer_range       ,    )

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
       def    A				(     self						:     int    ):



           '''simple docstring'''

           (
               (
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					
           )      	=		self.prepare_config_and_inputs()

           UpperCAmelCase      	=		True
           UpperCAmelCase      	=		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
           UpperCAmelCase      	=		ids_tensor([self.batch_size, self.seq_length]       ,    vocab_size=2    )

           return (
               config,
               input_ids,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
               choice_labels,
               encoder_hidden_states,
               encoder_attention_mask,
           )
       def    A				(     self						:     List[str]       ,    lowercase						:     Optional[int]       ,    lowercase						:     Any       ,    lowercase						:     Dict       ,    lowercase						:     str       ,    lowercase						:     Optional[Any]       ,    lowercase						:     List[Any]       ,    lowercase						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		RemBertModel(config=lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase    )
           UpperCAmelCase      	=		model(lowercase       ,    token_type_ids=lowercase    )
           UpperCAmelCase      	=		model(lowercase    )
           self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.seq_length, self.hidden_size)    )
       def    A				(     self						:     Optional[Any]       ,    lowercase						:     Tuple       ,    lowercase						:     Dict       ,    lowercase						:     Optional[int]       ,    lowercase						:     List[str]       ,    lowercase						:     List[str]       ,    lowercase						:     Optional[Any]       ,    lowercase						:     Any       ,    lowercase						:     Dict       ,    lowercase						:     Union[str, Any]       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		True
           UpperCAmelCase      	=		RemBertModel(lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    encoder_hidden_states=lowercase       ,    encoder_attention_mask=lowercase       ,    )
           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    encoder_hidden_states=lowercase       ,    )
           UpperCAmelCase      	=		model(lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase    )
           self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.seq_length, self.hidden_size)    )
       def    A				(     self						:     List[Any]       ,    lowercase						:     int       ,    lowercase						:     Union[str, Any]       ,    lowercase						:     str       ,    lowercase						:     Tuple       ,    lowercase						:     Optional[int]       ,    lowercase						:     List[str]       ,    lowercase						:     List[Any]       ,    lowercase						:     Any       ,    lowercase						:     Optional[Any]       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		RemBertForCausalLM(config=lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    labels=lowercase    )
           self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.vocab_size)    )
       def    A				(     self						:     str       ,    lowercase						:     List[str]       ,    lowercase						:     Tuple       ,    lowercase						:     List[Any]       ,    lowercase						:     Union[str, Any]       ,    lowercase						:     str       ,    lowercase						:     Any       ,    lowercase						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		RemBertForMaskedLM(config=lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    labels=lowercase    )
           self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.vocab_size)    )
       def    A				(     self						:     List[str]       ,    lowercase						:     Any       ,    lowercase						:     Any       ,    lowercase						:     Optional[Any]       ,    lowercase						:     Tuple       ,    lowercase						:     List[Any]       ,    lowercase						:     List[str]       ,    lowercase						:     int       ,    lowercase						:     str       ,    lowercase						:     Any       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		True
           UpperCAmelCase      	=		True
           UpperCAmelCase      	=		RemBertForCausalLM(config=lowercase    )
           model.to(lowercase    )
           model.eval()

           # first forward pass
           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    encoder_hidden_states=lowercase       ,    encoder_attention_mask=lowercase       ,    use_cache=lowercase       ,    )
           UpperCAmelCase      	=		outputs.past_key_values

           # create hypothetical multiple next token and extent to next_input_ids
           UpperCAmelCase      	=		ids_tensor((self.batch_size, 3)       ,    config.vocab_size    )
           UpperCAmelCase      	=		ids_tensor((self.batch_size, 3)       ,    vocab_size=2    )

           # append to next input_ids and
           UpperCAmelCase      	=		torch.cat([input_ids, next_tokens]       ,    dim=-1    )
           UpperCAmelCase      	=		torch.cat([input_mask, next_mask]       ,    dim=-1    )

           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    encoder_hidden_states=lowercase       ,    encoder_attention_mask=lowercase       ,    output_hidden_states=lowercase       ,    )['''hidden_states'''][0]
           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    encoder_hidden_states=lowercase       ,    encoder_attention_mask=lowercase       ,    past_key_values=lowercase       ,    output_hidden_states=lowercase       ,    )['''hidden_states'''][0]

           # select random slice
           UpperCAmelCase      	=		ids_tensor((1,)       ,    output_from_past.shape[-1]    ).item()
           UpperCAmelCase      	=		output_from_no_past[:, -3:, random_slice_idx].detach()
           UpperCAmelCase      	=		output_from_past[:, :, random_slice_idx].detach()

           self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )

           # test that outputs are equal for slice
           self.parent.assertTrue(torch.allclose(lowercase       ,    lowercase       ,    atol=1E-3    )    )
       def    A				(     self						:     int       ,    lowercase						:     Any       ,    lowercase						:     List[str]       ,    lowercase						:     List[str]       ,    lowercase						:     Optional[int]       ,    lowercase						:     str       ,    lowercase						:     Union[str, Any]       ,    lowercase						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		RemBertForQuestionAnswering(config=lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    start_positions=lowercase       ,    end_positions=lowercase       ,    )
           self.parent.assertEqual(result.start_logits.shape       ,    (self.batch_size, self.seq_length)    )
           self.parent.assertEqual(result.end_logits.shape       ,    (self.batch_size, self.seq_length)    )
       def    A				(     self						:     Optional[Any]       ,    lowercase						:     str       ,    lowercase						:     str       ,    lowercase						:     Any       ,    lowercase						:     Any       ,    lowercase						:     str       ,    lowercase						:     Optional[Any]       ,    lowercase						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.num_labels
           UpperCAmelCase      	=		RemBertForSequenceClassification(lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    labels=lowercase    )
           self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.num_labels)    )
       def    A				(     self						:     str       ,    lowercase						:     Optional[int]       ,    lowercase						:     Any       ,    lowercase						:     Optional[int]       ,    lowercase						:     Union[str, Any]       ,    lowercase						:     Any       ,    lowercase						:     List[str]       ,    lowercase						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.num_labels
           UpperCAmelCase      	=		RemBertForTokenClassification(config=lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		model(lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    labels=lowercase    )
           self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.num_labels)    )
       def    A				(     self						:     int       ,    lowercase						:     str       ,    lowercase						:     List[str]       ,    lowercase						:     List[Any]       ,    lowercase						:     int       ,    lowercase						:     Optional[Any]       ,    lowercase						:     Union[str, Any]       ,    lowercase						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.num_choices
           UpperCAmelCase      	=		RemBertForMultipleChoice(config=lowercase    )
           model.to(lowercase    )
           model.eval()
           UpperCAmelCase      	=		input_ids.unsqueeze(1    ).expand(-1       ,    self.num_choices       ,    -1    ).contiguous()
           UpperCAmelCase      	=		token_type_ids.unsqueeze(1    ).expand(-1       ,    self.num_choices       ,    -1    ).contiguous()
           UpperCAmelCase      	=		input_mask.unsqueeze(1    ).expand(-1       ,    self.num_choices       ,    -1    ).contiguous()
           UpperCAmelCase      	=		model(
               lowercase       ,    attention_mask=lowercase       ,    token_type_ids=lowercase       ,    labels=lowercase       ,    )
           self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.num_choices)    )







       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.prepare_config_and_inputs()
           (
               (
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					
           )      	=		config_and_inputs
           UpperCAmelCase      	=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
           return config, inputs_dict






@require_torch
class       _a  (     __a							,							__a							,							unittest.TestCase			):
       __a    :					Optional[int]           = (
           (
               RemBertModel,
               RemBertForMaskedLM,
               RemBertForCausalLM,
               RemBertForMultipleChoice,
               RemBertForQuestionAnswering,
               RemBertForSequenceClassification,
               RemBertForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       __a    :					Union[str, Any]           = (RemBertForCausalLM,) if is_torch_available() else ()
       __a    :					List[Any]           = (
           {
               """feature-extraction""": RemBertModel,
               """fill-mask""": RemBertForMaskedLM,
               """question-answering""": RemBertForQuestionAnswering,
               """text-classification""": RemBertForSequenceClassification,
               """text-generation""": RemBertForCausalLM,
               """token-classification""": RemBertForTokenClassification,
               """zero-shot""": RemBertForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		RemBertModelTester(self    )
           UpperCAmelCase      	=		ConfigTester(self       ,    config_class=lowercase       ,    hidden_size=37    )
       def    A				(     self						:     int    ):



           '''simple docstring'''

           self.config_tester.run_common_tests()
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*lowercase    )
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
               UpperCAmelCase      	=		type
               self.model_tester.create_and_check_model(*lowercase    )
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*lowercase    )
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*lowercase    )
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs_for_decoder()
           self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowercase    )
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*lowercase    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*lowercase    )
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*lowercase    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.model_tester.prepare_config_and_inputs_for_decoder()
           self.model_tester.create_and_check_model_as_decoder(*lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           (
               (
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					(
               UpperCAmelCase
           )	,					
           )      	=		self.model_tester.prepare_config_and_inputs_for_decoder()

           UpperCAmelCase      	=		None

           self.model_tester.create_and_check_model_as_decoder(
               lowercase       ,    lowercase       ,    lowercase       ,    lowercase       ,    lowercase       ,    lowercase       ,    lowercase       ,    lowercase       ,    lowercase       ,    )







       @slow
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCAmelCase      	=		RemBertModel.from_pretrained(lowercase    )
               self.assertIsNotNone(lowercase    )






@require_torch
class       _a  (     unittest.TestCase			):
       @slow
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		RemBertModel.from_pretrained('''google/rembert'''    )
           UpperCAmelCase      	=		torch.tensor([[312, 56_498, 313, 2_125, 313]]    )
           UpperCAmelCase      	=		torch.tensor([[0, 0, 0, 1, 1]]    )
           with torch.no_grad():
               UpperCAmelCase      	=		model(lowercase       ,    token_type_ids=lowercase       ,    output_hidden_states=lowercase    )

           UpperCAmelCase      	=		1_152

           UpperCAmelCase      	=		torch.Size((1, 5, hidden_size)    )
           self.assertEqual(output['''last_hidden_state'''].shape       ,    lowercase    )

           UpperCAmelCase      	=		torch.tensor(
               [
                   [
                       [0.0754, -0.2022, 0.1904],
                       [-0.3354, -0.3692, -0.4791],
                       [-0.2314, -0.6729, -0.0749],
                       [-0.0396, -0.3105, -0.4234],
                       [-0.1571, -0.0525, 0.5353],
                   ]
               ]    )

           # Running on the original tf implementation gives slightly different results here.
           # Not clear why this variations is present
           # TODO: Find reason for discrepancy
           # expected_original_implementation = [[
           #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
           #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
           #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
           #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
           #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
           # ]]

           self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]       ,    lowercase       ,    atol=1E-4    )    )










'''simple docstring'''







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       _a  (     __a							,							unittest.TestCase			):
       __a    :					Dict           = RealmTokenizer
       __a    :					Dict           = RealmTokenizerFast
       __a    :					Union[str, Any]           = True
       __a    :					Optional[int]           = True
       __a    :					int           = filter_non_english
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           super().setUp()

           UpperCAmelCase      	=		[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           UpperCAmelCase      	=		os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES['''vocab_file''']    )
           with open(self.vocab_file       ,    '''w'''       ,    encoding='''utf-8'''    ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
       def    A				(     self						:     int       ,    lowercase						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''
           UpperCAmelCase      	=		'''unwanted, running'''
           return input_text, output_text
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		tokenizer.tokenize('''UNwant\u00E9d,running'''    )
           self.assertListEqual(lowercase       ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )       ,    [9, 6, 7, 12, 10, 11]    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           if not self.test_rust_tokenizer:
               return

           UpperCAmelCase      	=		self.get_tokenizer()
           UpperCAmelCase      	=		self.get_rust_tokenizer()

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           # With lower casing
           UpperCAmelCase      	=		self.get_tokenizer(do_lower_case=lowercase    )
           UpperCAmelCase      	=		self.get_rust_tokenizer(do_lower_case=lowercase    )

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    )       ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''h\u00E9llo''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    never_split=['''[UNK]''']    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           UpperCAmelCase      	=		{}
           for i, token in enumerate(lowercase    ):
               UpperCAmelCase      	=		i
           UpperCAmelCase      	=		WordpieceTokenizer(vocab=lowercase       ,    unk_token='''[UNK]'''    )

           self.assertListEqual(tokenizer.tokenize(''''''    )       ,    []    )

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''    )       ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    )       ,    ['''[UNK]''', '''runn''', '''##ing''']    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           self.assertTrue(_is_whitespace(''' '''    )    )
           self.assertTrue(_is_whitespace('''\t'''    )    )
           self.assertTrue(_is_whitespace('''\r'''    )    )
           self.assertTrue(_is_whitespace('''\n'''    )    )
           self.assertTrue(_is_whitespace('''\u00A0'''    )    )

           self.assertFalse(_is_whitespace('''A'''    )    )
           self.assertFalse(_is_whitespace('''-'''    )    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           self.assertTrue(_is_control('''\u0005'''    )    )

           self.assertFalse(_is_control('''A'''    )    )
           self.assertFalse(_is_control(''' '''    )    )
           self.assertFalse(_is_control('''\t'''    )    )
           self.assertFalse(_is_control('''\r'''    )    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           self.assertTrue(_is_punctuation('''-'''    )    )
           self.assertTrue(_is_punctuation('''$'''    )    )
           self.assertTrue(_is_punctuation('''`'''    )    )
           self.assertTrue(_is_punctuation('''.'''    )    )

           self.assertFalse(_is_punctuation('''A'''    )    )
           self.assertFalse(_is_punctuation(''' '''    )    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )

           if self.test_rust_tokenizer:
               UpperCAmelCase      	=		self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )
       @slow
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class.from_pretrained('''bert-base-uncased'''    )

           UpperCAmelCase      	=		tokenizer.encode('''sequence builders'''       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		tokenizer.encode('''multi-sequence build'''       ,    add_special_tokens=lowercase    )

           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase    )
           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase       ,    lowercase    )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]
       def    A				(     self						:     int    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )

                   UpperCAmelCase      	=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                   UpperCAmelCase      	=		tokenizer_r.encode_plus(
                       lowercase       ,    return_attention_mask=lowercase       ,    return_token_type_ids=lowercase       ,    return_offsets_mapping=lowercase       ,    add_special_tokens=lowercase       ,    )

                   UpperCAmelCase      	=		tokenizer_r.do_lower_case if hasattr(lowercase       ,    '''do_lower_case'''    ) else False
                   UpperCAmelCase      	=		(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
                   self.assertEqual([e[0] for e in expected_results]       ,    tokens['''offset_mapping''']    )







       @slow
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		self.tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   UpperCAmelCase      	=		tokenizer_r.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )
                   UpperCAmelCase      	=		tokenizer_p.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )

                   UpperCAmelCase      	=		(2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape       ,    lowercase    )

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape       ,    lowercase    )




