




"""simple docstring"""

from __future__ import annotations

import unittest

from transformers import AlbertConfig, is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import TF_MODEL_FOR_PRETRAINING_MAPPING
							from transformers.models.albert.modeling_tf_albert import (
							    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
							    TFAlbertForMaskedLM,
							    TFAlbertForMultipleChoice,
							    TFAlbertForPreTraining,
							    TFAlbertForQuestionAnswering,
							    TFAlbertForSequenceClassification,
							    TFAlbertForTokenClassification,
							    TFAlbertModel,
							)

class     lowercase :







				def __init__(							self				:   List[str] ,       _lowerCamelCase				:   Optional[Any] ,       _lowerCamelCase				:   List[str]=13 ,       _lowerCamelCase				:   Union[str, Any]=7 ,       _lowerCamelCase				:   Optional[Any]=True ,       _lowerCamelCase				:   Dict=True ,       _lowerCamelCase				:   List[Any]=True ,       _lowerCamelCase				:   Any=True ,       _lowerCamelCase				:   List[str]=99 ,       _lowerCamelCase				:   Any=16 ,       _lowerCamelCase				:   Union[str, Any]=32 ,       _lowerCamelCase				:   Tuple=2 ,       _lowerCamelCase				:   Union[str, Any]=4 ,       _lowerCamelCase				:   Optional[int]=37 ,       _lowerCamelCase				:   List[str]="gelu" ,       _lowerCamelCase				:   str=0.1 ,       _lowerCamelCase				:   Dict=0.1 ,       _lowerCamelCase				:   Dict=5_12 ,       _lowerCamelCase				:   str=16 ,       _lowerCamelCase				:   Optional[int]=2 ,       _lowerCamelCase				:   List[Any]=0.02 ,       _lowerCamelCase				:   Union[str, Any]=3 ,       _lowerCamelCase				:   Union[str, Any]=4 ,       _lowerCamelCase				:   Dict=None ,       ):



					"""simple docstring"""

					A_					:			Tuple						=						parent
					A_					:			Dict						=						13
					A_					:			List[Any]						=						7
					A_					:			str						=						True
					A_					:			Dict						=						True
					A_					:			Optional[Any]						=						True
					A_					:			Union[str, Any]						=						True
					A_					:			Any						=						99
					A_					:			Any						=						16
					A_					:			str						=						32
					A_					:			Any						=						2
					A_					:			Optional[int]						=						4
					A_					:			Optional[int]						=						37
					A_					:			Optional[Any]						=						"gelu"
					A_					:			int						=						0.1
					A_					:			List[str]						=						0.1
					A_					:			Optional[int]						=						5_12
					A_					:			Union[str, Any]						=						16
					A_					:			Optional[int]						=						2
					A_					:			Tuple						=						0.02
					A_					:			str						=						3
					A_					:			Any						=						4
					A_					:			int						=						None







				def    a_					(							self				:   List[str]    ):



					"""simple docstring"""

					A_					:			Tuple						=						ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size    )

					A_					:			Tuple						=						None
					if self.use_input_mask:
						A_					:			List[str]						=						random_attention_mask([self.batch_size, self.seq_length]    )

					A_					:			Dict						=						None
					if self.use_token_type_ids:
						A_					:			List[Any]						=						ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size    )

					A_					:			Any						=						None
					A_					:			int						=						None
					A_					:			Tuple						=						None
					if self.use_labels:
						A_					:			List[Any]						=						ids_tensor([self.batch_size] ,       self.type_sequence_label_size    )
						A_					:			Optional[Any]						=						ids_tensor([self.batch_size, self.seq_length] ,       self.num_labels    )
						A_					:			Optional[int]						=						ids_tensor([self.batch_size] ,       self.num_choices    )

					A_					:			Optional[Any]						=						AlbertConfig(
					    vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       embedding_size=self.embedding_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       initializer_range=self.initializer_range ,       )

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   int ,       _lowerCamelCase				:   Union[str, Any] ,       _lowerCamelCase				:   Optional[Any] ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   List[str] ,       _lowerCamelCase				:   str    ):



					"""simple docstring"""

					A_					:			Dict						=						TFAlbertModel(config=SCREAMING_SNAKE_CASE__    )
					# inputs = {'input_ids': input_ids,
					#           'attention_mask': input_mask,
					#           'token_type_ids': token_type_ids}
					# sequence_output, pooled_output = model(**inputs)
					A_					:			int						=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
					A_					:			str						=						model(SCREAMING_SNAKE_CASE__    )

					A_					:			Any						=						[input_ids, input_mask]
					A_					:			Tuple						=						model(SCREAMING_SNAKE_CASE__    )

					A_					:			str						=						model(SCREAMING_SNAKE_CASE__    )

					self.parent.assertEqual(result.last_hidden_state.shape ,       (self.batch_size, self.seq_length, self.hidden_size)    )
					self.parent.assertEqual(result.pooler_output.shape ,       (self.batch_size, self.hidden_size)    )







				def    a_					(							self				:   Tuple ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   int ,       _lowerCamelCase				:   List[str] ,       _lowerCamelCase				:   Tuple ,       _lowerCamelCase				:   Union[str, Any] ,       _lowerCamelCase				:   Optional[int] ,       _lowerCamelCase				:   List[Any]    ):



					"""simple docstring"""

					A_					:			Any						=						self.num_labels
					A_					:			Union[str, Any]						=						TFAlbertForPreTraining(config=SCREAMING_SNAKE_CASE__    )
					A_					:			Dict						=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
					A_					:			Tuple						=						model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.prediction_logits.shape ,       (self.batch_size, self.seq_length, self.vocab_size)    )
					self.parent.assertEqual(result.sop_logits.shape ,       (self.batch_size, self.num_labels)    )







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   List[str] ,       _lowerCamelCase				:   Any ,       _lowerCamelCase				:   Optional[int] ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			Dict						=						TFAlbertForMaskedLM(config=SCREAMING_SNAKE_CASE__    )
					A_					:			List[str]						=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
					A_					:			Union[str, Any]						=						model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.vocab_size)    )







				def    a_					(							self				:   Union[str, Any] ,       _lowerCamelCase				:   Tuple ,       _lowerCamelCase				:   Any ,       _lowerCamelCase				:   Optional[Any] ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   str ,       _lowerCamelCase				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						self.num_labels
					A_					:			Optional[int]						=						TFAlbertForSequenceClassification(config=SCREAMING_SNAKE_CASE__    )
					A_					:			Union[str, Any]						=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
					A_					:			int						=						model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_labels)    )







				def    a_					(							self				:   Union[str, Any] ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   Dict ,       _lowerCamelCase				:   Dict ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   Optional[int] ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						TFAlbertForQuestionAnswering(config=SCREAMING_SNAKE_CASE__    )
					A_					:			Union[str, Any]						=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
					A_					:			List[Any]						=						model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.start_logits.shape ,       (self.batch_size, self.seq_length)    )
					self.parent.assertEqual(result.end_logits.shape ,       (self.batch_size, self.seq_length)    )







				def    a_					(							self				:   Optional[Any] ,       _lowerCamelCase				:   Any ,       _lowerCamelCase				:   Optional[int] ,       _lowerCamelCase				:   Any ,       _lowerCamelCase				:   Optional[Any] ,       _lowerCamelCase				:   Any ,       _lowerCamelCase				:   int ,       _lowerCamelCase				:   Any    ):



					"""simple docstring"""

					A_					:			Any						=						self.num_choices
					A_					:			Dict						=						TFAlbertForMultipleChoice(config=SCREAMING_SNAKE_CASE__    )
					A_					:			Optional[int]						=						tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__ ,       1    ) ,       (1, self.num_choices, 1)    )
					A_					:			int						=						tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__ ,       1    ) ,       (1, self.num_choices, 1)    )
					A_					:			Union[str, Any]						=						tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__ ,       1    ) ,       (1, self.num_choices, 1)    )
					A_					:			int						=						{
					    "input_ids": multiple_choice_inputs_ids,
					    "attention_mask": multiple_choice_input_mask,
					    "token_type_ids": multiple_choice_token_type_ids,
					}
					A_					:			Optional[Any]						=						model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertListEqual(list(result['''logits'''].shape    ) ,       [self.batch_size, self.num_choices]    )







				def    a_					(							self				:   Union[str, Any] ,       _lowerCamelCase				:   Tuple ,       _lowerCamelCase				:   Dict ,       _lowerCamelCase				:   Union[str, Any] ,       _lowerCamelCase				:   Tuple ,       _lowerCamelCase				:   Union[str, Any] ,       _lowerCamelCase				:   Optional[Any] ,       _lowerCamelCase				:   str    ):



					"""simple docstring"""

					A_					:			List[str]						=						self.num_labels
					A_					:			str						=						TFAlbertForTokenClassification(config=SCREAMING_SNAKE_CASE__    )
					A_					:			Tuple						=						{
					    "input_ids": input_ids,
					    "attention_mask": input_mask,
					    "token_type_ids": token_type_ids,
					}
					A_					:			int						=						model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertListEqual(list(result['''logits'''].shape    ) ,       [self.batch_size, self.seq_length, self.num_labels]    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			List[str]						=						self.prepare_config_and_inputs()
					(
					    A_      
					)					:			Optional[int]						=						config_and_inputs
					A_					:			Any						=						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
					return config, inputs_dict

@require_tf
class     lowercase (      _a       ,		_a       ,		unittest.TestCase):
				__lowerCAmelCase      :      Tuple					    =				(
				    (
				        TFAlbertModel,
				        TFAlbertForPreTraining,
				        TFAlbertForMaskedLM,
				        TFAlbertForSequenceClassification,
				        TFAlbertForQuestionAnswering,
				        TFAlbertForTokenClassification,
				        TFAlbertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				__lowerCAmelCase      :      Optional[Any]					    =				(
				    {
				        '''feature-extraction''': TFAlbertModel,
				        '''fill-mask''': TFAlbertForMaskedLM,
				        '''question-answering''': TFAlbertForQuestionAnswering,
				        '''text-classification''': TFAlbertForSequenceClassification,
				        '''token-classification''': TFAlbertForTokenClassification,
				        '''zero-shot''': TFAlbertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				__lowerCAmelCase      :      List[Any]					    =				False
				__lowerCAmelCase      :      str					    =				False







				def    a_					(							self				:   int ,       _lowerCamelCase				:   List[Any] ,       _lowerCamelCase				:   List[str] ,       _lowerCamelCase				:   List[Any]=False    ):



					"""simple docstring"""

					A_					:			Any						=						super()._prepare_for_class(SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ ,       return_labels=SCREAMING_SNAKE_CASE__    )

					if return_labels:
						if model_class in get_values(SCREAMING_SNAKE_CASE__    ):
							A_					:			str						=						tf.zeros(self.model_tester.batch_size ,       dtype=tf.intaa    )

					return inputs_dict







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			int						=						TFAlbertModelTester(self    )
					A_					:			Dict						=						ConfigTester(self ,       config_class=SCREAMING_SNAKE_CASE__ ,       hidden_size=37    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					self.config_tester.run_common_tests()







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			int						=						self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_albert_model(*SCREAMING_SNAKE_CASE__    )







				def    a_					(							self				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			int						=						self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_albert_for_pretraining(*SCREAMING_SNAKE_CASE__    )







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			List[Any]						=						self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_albert_for_masked_lm(*SCREAMING_SNAKE_CASE__    )







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					A_					:			Any						=						self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_albert_for_multiple_choice(*SCREAMING_SNAKE_CASE__    )







				def    a_					(							self				:   Tuple    ):



					"""simple docstring"""

					A_					:			str						=						self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_albert_for_sequence_classification(*SCREAMING_SNAKE_CASE__    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_albert_for_question_answering(*SCREAMING_SNAKE_CASE__    )







				@slow
				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						A_					:			Tuple						=						TFAlbertModel.from_pretrained(SCREAMING_SNAKE_CASE__    )
						self.assertIsNotNone(SCREAMING_SNAKE_CASE__    )

@require_tf
class     lowercase (      unittest.TestCase):







				@slow
				def    a_					(							self				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			List[str]						=						TFAlbertForPreTraining.from_pretrained('''albert-base-v2'''    )
					A_					:			int						=						tf.constant([[0, 1, 2, 3, 4, 5]]    )
					A_					:			List[str]						=						model(SCREAMING_SNAKE_CASE__    )[0]

					A_					:			Any						=						[1, 6, 3_00_00]
					self.assertEqual(output.shape ,       SCREAMING_SNAKE_CASE__    )

					A_					:			Optional[int]						=						tf.constant(
					    [
					        [
					            [4.595668, 0.74462754, -1.818147],
					            [4.5954347, 0.7454184, -1.8188258],
					            [4.5954905, 0.7448235, -1.8182316],
					        ]
					    ]    )
					tf.debugging.assert_near(output[:, :3, :3] ,       SCREAMING_SNAKE_CASE__ ,       atol=1E-4    )











"""simple docstring"""

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     lowercase (      __UpperCAmelCase       ,		unittest.TestCase):
				__lowerCAmelCase      :      Any					    =				ElectraTokenizer
				__lowerCAmelCase      :      Optional[int]					    =				ElectraTokenizerFast
				__lowerCAmelCase      :      Any					    =				True
				__lowerCAmelCase      :      Tuple					    =				True
				__lowerCAmelCase      :      Optional[Any]					    =				filter_non_english







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					super().setUp()

					A_					:			int						=						[
					    '''[UNK]''',
					    '''[CLS]''',
					    '''[SEP]''',
					    '''[PAD]''',
					    '''[MASK]''',
					    '''want''',
					    '''##want''',
					    '''##ed''',
					    '''wa''',
					    '''un''',
					    '''runn''',
					    '''##ing''',
					    ''',''',
					    '''low''',
					    '''lowest''',
					]
					A_					:			str						=						os.path.join(self.tmpdirname ,       VOCAB_FILES_NAMES['''vocab_file''']    )
					with open(self.vocab_file ,       '''w''' ,       encoding='''utf-8'''    ) as vocab_writer:
						vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )







				def    a_					(							self				:   Any ,       _lowerCamelCase				:   List[str]    ):



					"""simple docstring"""

					A_					:			List[str]						=						'''UNwant\u00E9d,running'''
					A_					:			Optional[int]						=						'''unwanted, running'''
					return input_text, output_text







				def    a_					(							self				:   List[Any]    ):



					"""simple docstring"""

					A_					:			Tuple						=						self.tokenizer_class(self.vocab_file    )

					A_					:			Dict						=						tokenizer.tokenize('''UNwant\u00E9d,running'''    )
					self.assertListEqual(_lowerCamelCase ,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase    ) ,       [9, 6, 7, 12, 10, 11]    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					if not self.test_rust_tokenizer:
						return

					A_					:			Any						=						self.get_tokenizer()
					A_					:			Optional[Any]						=						self.get_rust_tokenizer()

					A_					:			Union[str, Any]						=						'''UNwant\u00E9d,running'''

					A_					:			Optional[int]						=						tokenizer.tokenize(_lowerCamelCase    )
					A_					:			int						=						rust_tokenizer.tokenize(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Optional[int]						=						tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					A_					:			Union[str, Any]						=						rust_tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Any						=						self.get_rust_tokenizer()
					A_					:			Optional[int]						=						tokenizer.encode(_lowerCamelCase    )
					A_					:			List[Any]						=						rust_tokenizer.encode(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					# With lower casing
					A_					:			List[str]						=						self.get_tokenizer(do_lower_case=_lowerCamelCase    )
					A_					:			Any						=						self.get_rust_tokenizer(do_lower_case=_lowerCamelCase    )

					A_					:			List[Any]						=						'''UNwant\u00E9d,running'''

					A_					:			List[str]						=						tokenizer.tokenize(_lowerCamelCase    )
					A_					:			List[str]						=						rust_tokenizer.tokenize(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			Optional[Any]						=						tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					A_					:			Union[str, Any]						=						rust_tokenizer.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

					A_					:			List[Any]						=						self.get_rust_tokenizer()
					A_					:			List[Any]						=						tokenizer.encode(_lowerCamelCase    )
					A_					:			Dict						=						rust_tokenizer.encode(_lowerCamelCase    )
					self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Union[str, Any]						=						BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    ) ,       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )







				def    a_					(							self				:   List[str]    ):



					"""simple docstring"""

					A_					:			List[str]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    ) ,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			List[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''h\u00E9llo''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    ) ,       ['''hello''']    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    ) ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					A_					:			Tuple						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					A_					:			Optional[Any]						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       strip_accents=_lowerCamelCase    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    ) ,       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )







				def    a_					(							self				:   int    ):



					"""simple docstring"""

					A_					:			int						=						BasicTokenizer(do_lower_case=_lowerCamelCase ,       never_split=['''[UNK]''']    )

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    ) ,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			str						=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

					A_					:			Any						=						{}
					for i, token in enumerate(_lowerCamelCase    ):
						A_					:			int						=						i
					A_					:			str						=						WordpieceTokenizer(vocab=_lowerCamelCase ,       unk_token='''[UNK]'''    )

					self.assertListEqual(tokenizer.tokenize(''''''    ) ,       []    )

					self.assertListEqual(tokenizer.tokenize('''unwanted running'''    ) ,       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

					self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    ) ,       ['''[UNK]''', '''runn''', '''##ing''']    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					self.assertTrue(_is_whitespace(''' '''    )    )
					self.assertTrue(_is_whitespace('''\t'''    )    )
					self.assertTrue(_is_whitespace('''\r'''    )    )
					self.assertTrue(_is_whitespace('''\n'''    )    )
					self.assertTrue(_is_whitespace('''\u00A0'''    )    )

					self.assertFalse(_is_whitespace('''A'''    )    )
					self.assertFalse(_is_whitespace('''-'''    )    )







				def    a_					(							self				:   Any    ):



					"""simple docstring"""

					self.assertTrue(_is_control('''\u0005'''    )    )

					self.assertFalse(_is_control('''A'''    )    )
					self.assertFalse(_is_control(''' '''    )    )
					self.assertFalse(_is_control('''\t'''    )    )
					self.assertFalse(_is_control('''\r'''    )    )







				def    a_					(							self				:   Optional[int]    ):



					"""simple docstring"""

					self.assertTrue(_is_punctuation('''-'''    )    )
					self.assertTrue(_is_punctuation('''$'''    )    )
					self.assertTrue(_is_punctuation('''`'''    )    )
					self.assertTrue(_is_punctuation('''.'''    )    )

					self.assertFalse(_is_punctuation('''A'''    )    )
					self.assertFalse(_is_punctuation(''' '''    )    )







				def    a_					(							self				:   Dict    ):



					"""simple docstring"""

					A_					:			Optional[int]						=						self.get_tokenizer()
					A_					:			int						=						self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(_lowerCamelCase    ) for t in ['''Test''', '''\xad''', '''test''']] ,       [['''[UNK]'''], [], ['''[UNK]''']]    )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(_lowerCamelCase    ) for t in ['''Test''', '''\xad''', '''test''']] ,       [['''[UNK]'''], [], ['''[UNK]''']]    )







				@slow
				def    a_					(							self				:   Union[str, Any]    ):



					"""simple docstring"""

					A_					:			List[Any]						=						self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''    )

					A_					:			Tuple						=						tokenizer.encode('''sequence builders''' ,       add_special_tokens=_lowerCamelCase    )
					A_					:			List[Any]						=						tokenizer.encode('''multi-sequence build''' ,       add_special_tokens=_lowerCamelCase    )

					A_					:			List[Any]						=						tokenizer.build_inputs_with_special_tokens(_lowerCamelCase    )
					A_					:			Any						=						tokenizer.build_inputs_with_special_tokens(_lowerCamelCase ,       _lowerCamelCase    )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







				def    a_					(							self				:   str    ):



					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
							A_					:			Optional[Any]						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			Optional[int]						=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
							A_					:			Optional[int]						=						tokenizer_r.encode_plus(
							    _lowerCamelCase ,       return_attention_mask=_lowerCamelCase ,       return_token_type_ids=_lowerCamelCase ,       return_offsets_mapping=_lowerCamelCase ,       add_special_tokens=_lowerCamelCase ,       )

							A_					:			Union[str, Any]						=						tokenizer_r.do_lower_case if hasattr(_lowerCamelCase ,       '''do_lower_case'''    ) else False
							A_					:			List[Any]						=						(
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''A'''),
							        ((1, 2), ''','''),
							        ((3, 5), '''na'''),
							        ((5, 6), '''##ï'''),
							        ((6, 8), '''##ve'''),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), '''Allen'''),
							        ((21, 23), '''##NL'''),
							        ((23, 24), '''##P'''),
							        ((25, 33), '''sentence'''),
							        ((33, 34), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), '''a'''),
							        ((1, 2), ''','''),
							        ((3, 8), '''naive'''),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), '''allen'''),
							        ((21, 23), '''##nl'''),
							        ((23, 24), '''##p'''),
							        ((25, 33), '''sentence'''),
							        ((33, 34), '''.'''),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results] ,       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
							self.assertEqual([e[0] for e in expected_results] ,       tokens['''offset_mapping''']    )







				def    a_					(							self				:   Optional[Any]    ):



					"""simple docstring"""

					A_					:			str						=						['''的''', '''人''', '''有''']
					A_					:			Optional[int]						=						''''''.join(_lowerCamelCase    )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
							A_					:			List[Any]						=						True
							A_					:			Any						=						self.tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )
							A_					:			str						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			List[Any]						=						tokenizer_p.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
							A_					:			Any						=						tokenizer_r.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )

							A_					:			Union[str, Any]						=						tokenizer_r.convert_ids_to_tokens(_lowerCamelCase    )
							A_					:			List[Any]						=						tokenizer_p.convert_ids_to_tokens(_lowerCamelCase    )

							# it is expected that each Chinese character is not preceded by "##"
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )

							A_					:			List[Any]						=						False
							A_					:			Any						=						self.rust_tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )
							A_					:			Union[str, Any]						=						self.tokenizer_class.from_pretrained(_lowerCamelCase ,       **_lowerCamelCase    )

							A_					:			Optional[Any]						=						tokenizer_r.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )
							A_					:			int						=						tokenizer_p.encode(_lowerCamelCase ,       add_special_tokens=_lowerCamelCase    )

							A_					:			str						=						tokenizer_r.convert_ids_to_tokens(_lowerCamelCase    )
							A_					:			List[Any]						=						tokenizer_p.convert_ids_to_tokens(_lowerCamelCase    )

							# it is expected that only the first Chinese character is not preceded by "##".
							A_					:			Dict						=						[
							    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase    )
							]
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )
							self.assertListEqual(_lowerCamelCase ,       _lowerCamelCase    )





