






import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    YosoForMaskedLM,
							    YosoForMultipleChoice,
							    YosoForQuestionAnswering,
							    YosoForSequenceClassification,
							    YosoForTokenClassification,
							    YosoModel,
							)
							from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST


class 					__SCREAMING_SNAKE_CASE:
							def __init__(       self: Dict    ,		UpperCamelCase: str    ,		UpperCamelCase: int=13    ,		UpperCamelCase: Any=7    ,		UpperCamelCase: List[str]=True    ,		UpperCamelCase: int=True    ,		UpperCamelCase: Dict=True    ,		UpperCamelCase: Tuple=True    ,		UpperCamelCase: Tuple=99    ,		UpperCamelCase: Optional[int]=32    ,		UpperCamelCase: List[str]=5    ,		UpperCamelCase: Any=4    ,		UpperCamelCase: int=37    ,		UpperCamelCase: str="gelu"    ,		UpperCamelCase: List[str]=0.1    ,		UpperCamelCase: Optional[int]=0.1    ,		UpperCamelCase: Tuple=5_12    ,		UpperCamelCase: int=16    ,		UpperCamelCase: List[str]=2    ,		UpperCamelCase: Tuple=0.02    ,		UpperCamelCase: int=3    ,		UpperCamelCase: Optional[int]=4    ,		UpperCamelCase: Dict=None    ,		)						->						List[Any]:
											snake_case__													=       parent
											snake_case__													=       batch_size
											snake_case__													=       seq_length
											snake_case__													=       is_training
											snake_case__													=       use_input_mask
											snake_case__													=       use_token_type_ids
											snake_case__													=       use_labels
											snake_case__													=       vocab_size
											snake_case__													=       hidden_size
											snake_case__													=       num_hidden_layers
											snake_case__													=       num_attention_heads
											snake_case__													=       intermediate_size
											snake_case__													=       hidden_act
											snake_case__													=       hidden_dropout_prob
											snake_case__													=       attention_probs_dropout_prob
											snake_case__													=       max_position_embeddings
											snake_case__													=       type_vocab_size
											snake_case__													=       type_sequence_label_size
											snake_case__													=       initializer_range
											snake_case__													=       num_labels
											snake_case__													=       num_choices
											snake_case__													=       scope
							def lowerCAmelCase_	(       self: int  )						->						str:
											snake_case__													=       ids_tensor([self.batch_size, self.seq_length]    ,		self.vocab_size  )

											snake_case__													=       None
											if self.use_input_mask:
															snake_case__													=       random_attention_mask([self.batch_size, self.seq_length]  )

											snake_case__													=       None
											if self.use_token_type_ids:
															snake_case__													=       ids_tensor([self.batch_size, self.seq_length]    ,		self.type_vocab_size  )

											snake_case__													=       None
											snake_case__													=       None
											snake_case__													=       None
											if self.use_labels:
															snake_case__													=       ids_tensor([self.batch_size]    ,		self.type_sequence_label_size  )
															snake_case__													=       ids_tensor([self.batch_size, self.seq_length]    ,		self.num_labels  )
															snake_case__													=       ids_tensor([self.batch_size]    ,		self.num_choices  )

											snake_case__													=       self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
							def lowerCAmelCase_	(       self: List[Any]  )						->						Union[str, Any]:
											return YosoConfig(
											    vocab_size=self.vocab_size    ,		hidden_size=self.hidden_size    ,		num_hidden_layers=self.num_hidden_layers    ,		num_attention_heads=self.num_attention_heads    ,		intermediate_size=self.intermediate_size    ,		hidden_act=self.hidden_act    ,		hidden_dropout_prob=self.hidden_dropout_prob    ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,		max_position_embeddings=self.max_position_embeddings    ,		type_vocab_size=self.type_vocab_size    ,		is_decoder=SCREAMING_SNAKE_CASE__    ,		initializer_range=self.initializer_range    ,		)
							def lowerCAmelCase_	(       self: Union[str, Any]  )						->						str:
											snake_case__													=       self.get_config()
											snake_case__													=       3_00
											return config
							def lowerCAmelCase_	(       self: Tuple  )						->						Union[str, Any]:
											(
											    snake_case__      
											)													=       self.prepare_config_and_inputs()

											snake_case__													=       True
											snake_case__													=       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
											snake_case__													=       ids_tensor([self.batch_size, self.seq_length]    ,		vocab_size=2  )

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)
							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: Optional[int]    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: str    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: str    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: Any  )						->						Union[str, Any]:
											snake_case__													=       YosoModel(config=SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       model(SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__  )
											snake_case__													=       model(SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__  )
											snake_case__													=       model(SCREAMING_SNAKE_CASE__  )
											self.parent.assertEqual(result.last_hidden_state.shape    ,		(self.batch_size, self.seq_length, self.hidden_size)  )
							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: str    ,		UpperCamelCase: str    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: str    ,		UpperCamelCase: Any    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: str    ,		)						->						List[Any]:
											snake_case__													=       True
											snake_case__													=       YosoModel(SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       model(
											    SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		encoder_hidden_states=SCREAMING_SNAKE_CASE__    ,		encoder_attention_mask=SCREAMING_SNAKE_CASE__    ,		)
											snake_case__													=       model(
											    SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		encoder_hidden_states=SCREAMING_SNAKE_CASE__    ,		)
											snake_case__													=       model(SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__  )
											self.parent.assertEqual(result.last_hidden_state.shape    ,		(self.batch_size, self.seq_length, self.hidden_size)  )
							def lowerCAmelCase_	(       self: int    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: Dict    ,		UpperCamelCase: int    ,		UpperCamelCase: Union[str, Any]    ,		UpperCamelCase: Any    ,		UpperCamelCase: str    ,		UpperCamelCase: Union[str, Any]  )						->						List[str]:
											snake_case__													=       YosoForMaskedLM(config=SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       model(SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		labels=SCREAMING_SNAKE_CASE__  )
											self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.seq_length, self.vocab_size)  )
							def lowerCAmelCase_	(       self: Any    ,		UpperCamelCase: List[Any]    ,		UpperCamelCase: int    ,		UpperCamelCase: Dict    ,		UpperCamelCase: Union[str, Any]    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Optional[int]  )						->						int:
											snake_case__													=       YosoForQuestionAnswering(config=SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       model(
											    SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		start_positions=SCREAMING_SNAKE_CASE__    ,		end_positions=SCREAMING_SNAKE_CASE__    ,		)
											self.parent.assertEqual(result.start_logits.shape    ,		(self.batch_size, self.seq_length)  )
											self.parent.assertEqual(result.end_logits.shape    ,		(self.batch_size, self.seq_length)  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: Optional[Any]    ,		UpperCamelCase: int    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Optional[int]    ,		UpperCamelCase: Optional[int]    ,		UpperCamelCase: Union[str, Any]    ,		UpperCamelCase: List[Any]  )						->						int:
											snake_case__													=       self.num_labels
											snake_case__													=       YosoForSequenceClassification(SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       model(SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		labels=SCREAMING_SNAKE_CASE__  )
											self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.num_labels)  )
							def lowerCAmelCase_	(       self: int    ,		UpperCamelCase: str    ,		UpperCamelCase: Optional[int]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: List[Any]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Any  )						->						Union[str, Any]:
											snake_case__													=       self.num_labels
											snake_case__													=       YosoForTokenClassification(config=SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       model(SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		labels=SCREAMING_SNAKE_CASE__  )
											self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.seq_length, self.num_labels)  )
							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Tuple    ,		UpperCamelCase: Dict    ,		UpperCamelCase: Union[str, Any]    ,		UpperCamelCase: Optional[int]    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: List[Any]  )						->						str:
											snake_case__													=       self.num_choices
											snake_case__													=       YosoForMultipleChoice(config=SCREAMING_SNAKE_CASE__  )
											model.to(SCREAMING_SNAKE_CASE__  )
											model.eval()
											snake_case__													=       input_ids.unsqueeze(1  ).expand(-1    ,		self.num_choices    ,		-1  ).contiguous()
											snake_case__													=       token_type_ids.unsqueeze(1  ).expand(-1    ,		self.num_choices    ,		-1  ).contiguous()
											snake_case__													=       input_mask.unsqueeze(1  ).expand(-1    ,		self.num_choices    ,		-1  ).contiguous()
											snake_case__													=       model(
											    SCREAMING_SNAKE_CASE__    ,		attention_mask=SCREAMING_SNAKE_CASE__    ,		token_type_ids=SCREAMING_SNAKE_CASE__    ,		labels=SCREAMING_SNAKE_CASE__    ,		)
											self.parent.assertEqual(result.logits.shape    ,		(self.batch_size, self.num_choices)  )


							def lowerCAmelCase_	(       self: List[Any]  )						->						Union[str, Any]:
											snake_case__													=       self.prepare_config_and_inputs()
											(
											    snake_case__      
											)													=       config_and_inputs
											snake_case__													=       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
											return config, inputs_dict


@require_torch
class 					__SCREAMING_SNAKE_CASE(		_a      ,					_a      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    (
							    (
							        YosoModel,
							        YosoForMaskedLM,
							        YosoForMultipleChoice,
							        YosoForQuestionAnswering,
							        YosoForSequenceClassification,
							        YosoForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							_UpperCAmelCase	      =    False
							_UpperCAmelCase	      =    False
							_UpperCAmelCase	      =    False

							_UpperCAmelCase	      =    ()
							_UpperCAmelCase	      =    (
							    {
							        '''feature-extraction''': YosoModel,
							        '''fill-mask''': YosoForMaskedLM,
							        '''question-answering''': YosoForQuestionAnswering,
							        '''text-classification''': YosoForSequenceClassification,
							        '''token-classification''': YosoForTokenClassification,
							        '''zero-shot''': YosoForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							def lowerCAmelCase_	(       self: Dict  )						->						List[str]:
											snake_case__													=       YosoModelTester(self  )
											snake_case__													=       ConfigTester(self    ,		config_class=SCREAMING_SNAKE_CASE__    ,		hidden_size=37  )
							def lowerCAmelCase_	(       self: List[Any]  )						->						Any:
											self.config_tester.run_common_tests()
							def lowerCAmelCase_	(       self: List[str]  )						->						Union[str, Any]:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__  )
							def lowerCAmelCase_	(       self: str  )						->						List[Any]:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
															snake_case__													=       type
															self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__  )
							def lowerCAmelCase_	(       self: int  )						->						Optional[int]:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE__  )
							def lowerCAmelCase_	(       self: Any  )						->						Tuple:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE__  )
							def lowerCAmelCase_	(       self: Any  )						->						int:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE__  )
							def lowerCAmelCase_	(       self: List[Any]  )						->						Union[str, Any]:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE__  )
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						int:
											snake_case__													=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE__  )
							@slow
							def lowerCAmelCase_	(       self: Tuple  )						->						int:
											for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															snake_case__													=       YosoModel.from_pretrained(SCREAMING_SNAKE_CASE__  )
															self.assertIsNotNone(SCREAMING_SNAKE_CASE__  )


							def lowerCAmelCase_	(       self: Any  )						->						int:
											return


@require_torch
class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							@slow
							def lowerCAmelCase_	(       self: int  )						->						List[str]:
											snake_case__													=       YosoModel.from_pretrained('uw-madison/yoso-4096'  )
											snake_case__													=       torch.tensor([[0, 1, 2, 3, 4, 5]]  )

											with torch.no_grad():
															snake_case__													=       model(SCREAMING_SNAKE_CASE__  )[0]

											snake_case__													=       torch.Size((1, 6, 7_68)  )
											self.assertEqual(output.shape    ,		SCREAMING_SNAKE_CASE__  )

											snake_case__													=       torch.tensor(
											    [[[-0.0_611, 0.1_242, 0.0_840], [0.0_280, -0.0_048, 0.1_125], [0.0_106, 0.0_226, 0.0_751]]]  )

											self.assertTrue(torch.allclose(output[:, :3, :3]    ,		SCREAMING_SNAKE_CASE__    ,		atol=1e-4  )  )
							@slow
							def lowerCAmelCase_	(       self: List[str]  )						->						List[str]:
											snake_case__													=       YosoForMaskedLM.from_pretrained('uw-madison/yoso-4096'  )
											snake_case__													=       torch.tensor([[0, 1, 2, 3, 4, 5]]  )

											with torch.no_grad():
															snake_case__													=       model(SCREAMING_SNAKE_CASE__  )[0]

											snake_case__													=       5_02_65

											snake_case__													=       torch.Size((1, 6, vocab_size)  )
											self.assertEqual(output.shape    ,		SCREAMING_SNAKE_CASE__  )

											snake_case__													=       torch.tensor(
											    [[[-2.1_313, -3.7_285, -2.2_407], [-2.7_047, -3.3_314, -2.6_408], [0.0_629, -2.5_166, -0.3_356]]]  )

											self.assertTrue(torch.allclose(output[:, :3, :3]    ,		SCREAMING_SNAKE_CASE__    ,		atol=1e-4  )  )


							@slow
							def lowerCAmelCase_	(       self: List[Any]  )						->						Optional[int]:
											snake_case__													=       YosoForMaskedLM.from_pretrained('uw-madison/yoso-4096'  )
											snake_case__													=       torch.arange(40_96  ).unsqueeze(0  )

											with torch.no_grad():
															snake_case__													=       model(SCREAMING_SNAKE_CASE__  )[0]

											snake_case__													=       5_02_65

											snake_case__													=       torch.Size((1, 40_96, vocab_size)  )
											self.assertEqual(output.shape    ,		SCREAMING_SNAKE_CASE__  )

											snake_case__													=       torch.tensor(
											    [[[-2.3_914, -4.3_742, -5.0_956], [-4.0_988, -4.2_384, -7.0_406], [-3.1_427, -3.7_192, -6.6_800]]]  )

											self.assertTrue(torch.allclose(output[:, :3, :3]    ,		SCREAMING_SNAKE_CASE__    ,		atol=1e-4  )  )












import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    ElectraTokenizer
							_UpperCAmelCase	      =    ElectraTokenizerFast
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    filter_non_english
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											super().setUp()

											snake_case__													=       [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case__													=       os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file    ,		'w'    ,		encoding='utf-8'  ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: List[str]  )						->						Union[str, Any]:
											snake_case__													=       'UNwant\u00E9d,running'
											snake_case__													=       'unwanted, running'
											return input_text, output_text
							def lowerCAmelCase_	(       self: List[str]  )						->						List[Any]:
											snake_case__													=       self.tokenizer_class(self.vocab_file  )

											snake_case__													=       tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(UpperCamelCase    ,		['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase  )    ,		[9, 6, 7, 12, 10, 11]  )
							def lowerCAmelCase_	(       self: str  )						->						Tuple:
											if not self.test_rust_tokenizer:
															return

											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											# With lower casing
											snake_case__													=       self.get_tokenizer(do_lower_case=UpperCamelCase  )
											snake_case__													=       self.get_rust_tokenizer(do_lower_case=UpperCamelCase  )

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											snake_case__													=       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )    ,		['ah', '\u535A', '\u63A8', 'zz']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[Any]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['h\u00E9llo']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: Any  )						->						int:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: str  )						->						List[str]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: int  )						->						str:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
							def lowerCAmelCase_	(       self: Any  )						->						Dict:
											snake_case__													=       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											snake_case__													=       {}
											for i, token in enumerate(UpperCamelCase  ):
															snake_case__													=       i
											snake_case__													=       WordpieceTokenizer(vocab=UpperCamelCase    ,		unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )    ,		[]  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )    ,		['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )    ,		['[UNK]', 'runn', '##ing']  )
							def lowerCAmelCase_	(       self: Dict  )						->						str:
											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Tuple:
											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
							def lowerCAmelCase_	(       self: str  )						->						Any:
											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )
							@slow
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											snake_case__													=       tokenizer.encode('sequence builders'    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       tokenizer.encode('multi-sequence build'    ,		add_special_tokens=UpperCamelCase  )

											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase  )
											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase    ,		UpperCamelCase  )

											assert encoded_sentence == [1_01] + text + [1_02]
											assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			snake_case__													=       tokenizer_r.encode_plus(
																			    UpperCamelCase    ,		return_attention_mask=UpperCamelCase    ,		return_token_type_ids=UpperCamelCase    ,		return_offsets_mapping=UpperCamelCase    ,		add_special_tokens=UpperCamelCase    ,		)

																			snake_case__													=       tokenizer_r.do_lower_case if hasattr(UpperCamelCase    ,		'do_lower_case'  ) else False
																			snake_case__													=       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]    ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																			self.assertEqual([e[0] for e in expected_results]    ,		tokens['offset_mapping']  )


							def lowerCAmelCase_	(       self: Tuple  )						->						Any:
											snake_case__													=       ['的', '人', '有']
											snake_case__													=       ''.join(UpperCamelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       True
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

																			snake_case__													=       False
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			snake_case__													=       [
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase  )
																			]
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )




