






import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    PreTrainedTokenizerFast
							_UpperCAmelCase	      =    False
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    "tokenizer_file"
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Optional[Any]:
											snake_case__													=       False  # because we don't have pretrained_vocab_files_map
											super().setUp()
											snake_case__													=       True

											snake_case__													=       ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
											snake_case__													=       'SaulLu/dummy-tokenizer-bytelevel-bpe'

											# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
											snake_case__													=       [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

											snake_case__													=       PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
											tokenizer.save_pretrained(self.tmpdirname  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Dict:
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass
							def lowerCAmelCase_	(       self: str  )						->						Union[str, Any]:
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass
							def lowerCAmelCase_	(       self: Any  )						->						Tuple:
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass
							def lowerCAmelCase_	(       self: int  )						->						Optional[Any]:
											# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
											pass
							def lowerCAmelCase_	(       self: Tuple  )						->						Optional[int]:
											snake_case__													=       self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			try:
																							snake_case__													=       tempfile.mkdtemp()
																							snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																							tokenizer.save_pretrained(self.tmpdirname  )
																							super().test_training_new_tokenizer()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname  )
																							snake_case__													=       tmpdirname_orig
							def lowerCAmelCase_	(       self: Union[str, Any]  )						->						Optional[int]:
											snake_case__													=       self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			try:
																							snake_case__													=       tempfile.mkdtemp()
																							snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																							tokenizer.save_pretrained(self.tmpdirname  )
																							super().test_training_new_tokenizer_with_special_tokens_change()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname  )
																							snake_case__													=       tmpdirname_orig


							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Union[str, Any]:
											snake_case__													=       self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

											snake_case__													=       ('a' for _ in range(10_00  ))
											snake_case__													=       tokenizer.train_new_from_iterator(text_iterator=UpperCamelCase    ,		length=10_00    ,		vocab_size=50  )

											snake_case__													=       new_tokenizer.encode('a🤗'  )
											self.assertEqual(UpperCamelCase    ,		[64, 1_72, 2_53, 97, 2_45]  )


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def lowerCAmelCase_	(       self: List[Any]  )						->						Optional[int]:
											snake_case__													=       AutoTokenizer.from_pretrained('bert-base-cased'  )
											snake_case__													=       json.loads(tokenizer._tokenizer.to_str()  )
											snake_case__													=       len(UpperCamelCase  )

											with tempfile.TemporaryDirectory() as tmp_dir:
															# Hack to save this in the tokenizer_config.json
															snake_case__													=       ['tokenizer.4.0.0.json']
															tokenizer.save_pretrained(UpperCamelCase  )
															json.dump(UpperCamelCase    ,		open(os.path.join(UpperCamelCase    ,		'tokenizer.4.0.0.json'  )    ,		'w'  )  )

															# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
															snake_case__													=       AutoTokenizer.from_pretrained(UpperCamelCase  )
															self.assertEqual(len(UpperCamelCase  )    ,		len(UpperCamelCase  ) + 1  )
															snake_case__													=       json.loads(new_tokenizer._tokenizer.to_str()  )
															self.assertIn('huggingface'    ,		json_tokenizer['model']['vocab']  )

															# Will need to be adjusted if we reach v42 and this test is still here.
															# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
															shutil.move(os.path.join(UpperCamelCase    ,		'tokenizer.4.0.0.json'  )    ,		os.path.join(UpperCamelCase    ,		'tokenizer.42.0.0.json'  )  )
															snake_case__													=       ['tokenizer.42.0.0.json']
															tokenizer.save_pretrained(UpperCamelCase  )
															snake_case__													=       AutoTokenizer.from_pretrained(UpperCamelCase  )
															self.assertEqual(len(UpperCamelCase  )    ,		len(UpperCamelCase  )  )
															snake_case__													=       json.loads(new_tokenizer._tokenizer.to_str()  )
															self.assertNotIn('huggingface'    ,		json_tokenizer['model']['vocab']  )


							def lowerCAmelCase_	(       self: Any  )						->						Tuple:
											# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
											snake_case__													=       'hf-internal-testing/test-two-tokenizers'

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											snake_case__													=       AutoTokenizer.from_pretrained(UpperCamelCase  )
											self.assertEqual(len(UpperCamelCase  )    ,		2_89_97  )
											snake_case__													=       json.loads(tokenizer._tokenizer.to_str()  )
											self.assertIn('huggingface'    ,		json_tokenizer['model']['vocab']  )

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											snake_case__													=       '3.0.0'
											snake_case__													=       old_transformers.models.auto.AutoTokenizer.from_pretrained(UpperCamelCase  )
											self.assertEqual(len(UpperCamelCase  )    ,		2_89_96  )
											snake_case__													=       json.loads(old_tokenizer._tokenizer.to_str()  )
											self.assertNotIn('huggingface'    ,		json_tokenizer['model']['vocab']  )


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Any:
											# See https://github.com/huggingface/transformers/pull/12550
											# and https://github.com/huggingface/tokenizers/issues/537
											snake_case__													=       PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'  )
											snake_case__													=       'The Matrix is a 1999 science fiction action film.'

											with concurrent.futures.ThreadPoolExecutor() as executor:
															snake_case__													=       [executor.submit(self.fetch    ,		UpperCamelCase    ,		UpperCamelCase  ) for i in range(10  )]
															snake_case__													=       [future.result() for future in futures]
															self.assertEqual(UpperCamelCase    ,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )


							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: List[Any]  )						->						str:
											return tokenizer.encode(UpperCamelCase    ,		truncation='longest_first'    ,		padding='longest'  )












import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    ElectraTokenizer
							_UpperCAmelCase	      =    ElectraTokenizerFast
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    filter_non_english
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											super().setUp()

											snake_case__													=       [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case__													=       os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file    ,		'w'    ,		encoding='utf-8'  ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: List[str]  )						->						Union[str, Any]:
											snake_case__													=       'UNwant\u00E9d,running'
											snake_case__													=       'unwanted, running'
											return input_text, output_text
							def lowerCAmelCase_	(       self: List[str]  )						->						List[Any]:
											snake_case__													=       self.tokenizer_class(self.vocab_file  )

											snake_case__													=       tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(UpperCamelCase    ,		['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase  )    ,		[9, 6, 7, 12, 10, 11]  )
							def lowerCAmelCase_	(       self: str  )						->						Tuple:
											if not self.test_rust_tokenizer:
															return

											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											# With lower casing
											snake_case__													=       self.get_tokenizer(do_lower_case=UpperCamelCase  )
											snake_case__													=       self.get_rust_tokenizer(do_lower_case=UpperCamelCase  )

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											snake_case__													=       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )    ,		['ah', '\u535A', '\u63A8', 'zz']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[Any]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['h\u00E9llo']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: Any  )						->						int:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: str  )						->						List[str]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: int  )						->						str:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
							def lowerCAmelCase_	(       self: Any  )						->						Dict:
											snake_case__													=       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											snake_case__													=       {}
											for i, token in enumerate(UpperCamelCase  ):
															snake_case__													=       i
											snake_case__													=       WordpieceTokenizer(vocab=UpperCamelCase    ,		unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )    ,		[]  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )    ,		['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )    ,		['[UNK]', 'runn', '##ing']  )
							def lowerCAmelCase_	(       self: Dict  )						->						str:
											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Tuple:
											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
							def lowerCAmelCase_	(       self: str  )						->						Any:
											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )
							@slow
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											snake_case__													=       tokenizer.encode('sequence builders'    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       tokenizer.encode('multi-sequence build'    ,		add_special_tokens=UpperCamelCase  )

											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase  )
											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase    ,		UpperCamelCase  )

											assert encoded_sentence == [1_01] + text + [1_02]
											assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			snake_case__													=       tokenizer_r.encode_plus(
																			    UpperCamelCase    ,		return_attention_mask=UpperCamelCase    ,		return_token_type_ids=UpperCamelCase    ,		return_offsets_mapping=UpperCamelCase    ,		add_special_tokens=UpperCamelCase    ,		)

																			snake_case__													=       tokenizer_r.do_lower_case if hasattr(UpperCamelCase    ,		'do_lower_case'  ) else False
																			snake_case__													=       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]    ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																			self.assertEqual([e[0] for e in expected_results]    ,		tokens['offset_mapping']  )


							def lowerCAmelCase_	(       self: Tuple  )						->						Any:
											snake_case__													=       ['的', '人', '有']
											snake_case__													=       ''.join(UpperCamelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       True
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

																			snake_case__													=       False
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			snake_case__													=       [
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase  )
																			]
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )




