



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 		UpperCAmelCase__      (      __UpperCamelCase						,unittest.TestCase       ):




							'''simple docstring'''






							UpperCamelCase      						=	ElectraTokenizer
							UpperCamelCase      						=	ElectraTokenizerFast
							UpperCamelCase      						=	True
							UpperCamelCase      						=	True
							UpperCamelCase      						=	filter_non_english




							def 			snake_case__					(       self							:			List[str]			):
									'''simple docstring'''





									super().setUp()

									__UpperCAmelCase   :					Any													=     [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									__UpperCAmelCase   :					Dict													=     os.path.join(self.tmpdirname	,	VOCAB_FILES_NAMES['''vocab_file''']			)
									with open(self.vocab_file	,	'''w'''	,	encoding='''utf-8'''			) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)




							def 			snake_case__					(       self							:			Tuple	,	a_							:			List[str]			):
									'''simple docstring'''





									__UpperCAmelCase   :					Optional[int]													=     '''UNwant\u00E9d,running'''
									__UpperCAmelCase   :					List[str]													=     '''unwanted, running'''
									return input_text, output_text




							def 			snake_case__					(       self							:			str			):
									'''simple docstring'''





									__UpperCAmelCase   :					Union[str, Any]													=     self.tokenizer_class(self.vocab_file			)

									__UpperCAmelCase   :					int													=     tokenizer.tokenize('''UNwant\u00E9d,running'''			)
									self.assertListEqual(a_	,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)	,	[9, 6, 7, 12, 10, 11]			)




							def 			snake_case__					(       self							:			List[Any]			):
									'''simple docstring'''





									if not self.test_rust_tokenizer:
											return

									__UpperCAmelCase   :					str													=     self.get_tokenizer()
									__UpperCAmelCase   :					Tuple													=     self.get_rust_tokenizer()

									__UpperCAmelCase   :					str													=     '''UNwant\u00E9d,running'''

									__UpperCAmelCase   :					int													=     tokenizer.tokenize(a_			)
									__UpperCAmelCase   :					int													=     rust_tokenizer.tokenize(a_			)
									self.assertListEqual(a_	,	a_			)

									__UpperCAmelCase   :					Optional[Any]													=     tokenizer.encode(a_	,	add_special_tokens=a_			)
									__UpperCAmelCase   :					Union[str, Any]													=     rust_tokenizer.encode(a_	,	add_special_tokens=a_			)
									self.assertListEqual(a_	,	a_			)

									__UpperCAmelCase   :					str													=     self.get_rust_tokenizer()
									__UpperCAmelCase   :					Dict													=     tokenizer.encode(a_			)
									__UpperCAmelCase   :					List[Any]													=     rust_tokenizer.encode(a_			)
									self.assertListEqual(a_	,	a_			)

									# With lower casing
									__UpperCAmelCase   :					Tuple													=     self.get_tokenizer(do_lower_case=a_			)
									__UpperCAmelCase   :					Union[str, Any]													=     self.get_rust_tokenizer(do_lower_case=a_			)

									__UpperCAmelCase   :					Union[str, Any]													=     '''UNwant\u00E9d,running'''

									__UpperCAmelCase   :					List[str]													=     tokenizer.tokenize(a_			)
									__UpperCAmelCase   :					str													=     rust_tokenizer.tokenize(a_			)
									self.assertListEqual(a_	,	a_			)

									__UpperCAmelCase   :					Optional[int]													=     tokenizer.encode(a_	,	add_special_tokens=a_			)
									__UpperCAmelCase   :					int													=     rust_tokenizer.encode(a_	,	add_special_tokens=a_			)
									self.assertListEqual(a_	,	a_			)

									__UpperCAmelCase   :					List[Any]													=     self.get_rust_tokenizer()
									__UpperCAmelCase   :					str													=     tokenizer.encode(a_			)
									__UpperCAmelCase   :					List[str]													=     rust_tokenizer.encode(a_			)
									self.assertListEqual(a_	,	a_			)




							def 			snake_case__					(       self							:			Dict			):
									'''simple docstring'''





									__UpperCAmelCase   :					Optional[Any]													=     BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			)	,	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)




							def 			snake_case__					(       self							:			str			):
									'''simple docstring'''





									__UpperCAmelCase   :					List[str]													=     BasicTokenizer(do_lower_case=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)	,	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)	,	['''hello''']			)




							def 			snake_case__					(       self							:			Optional[Any]			):
									'''simple docstring'''





									__UpperCAmelCase   :					Any													=     BasicTokenizer(do_lower_case=a_	,	strip_accents=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)	,	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)	,	['''h\u00E9llo''']			)




							def 			snake_case__					(       self							:			str			):
									'''simple docstring'''





									__UpperCAmelCase   :					Tuple													=     BasicTokenizer(do_lower_case=a_	,	strip_accents=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)	,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)	,	['''hello''']			)




							def 			snake_case__					(       self							:			Dict			):
									'''simple docstring'''





									__UpperCAmelCase   :					Optional[Any]													=     BasicTokenizer(do_lower_case=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)	,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)	,	['''hello''']			)




							def 			snake_case__					(       self							:			Union[str, Any]			):
									'''simple docstring'''





									__UpperCAmelCase   :					Any													=     BasicTokenizer(do_lower_case=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			)	,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




							def 			snake_case__					(       self							:			Tuple			):
									'''simple docstring'''





									__UpperCAmelCase   :					Optional[int]													=     BasicTokenizer(do_lower_case=a_	,	strip_accents=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)	,	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




							def 			snake_case__					(       self							:			Optional[Any]			):
									'''simple docstring'''





									__UpperCAmelCase   :					List[str]													=     BasicTokenizer(do_lower_case=a_	,	strip_accents=a_			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			)	,	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)




							def 			snake_case__					(       self							:			Union[str, Any]			):
									'''simple docstring'''





									__UpperCAmelCase   :					Tuple													=     BasicTokenizer(do_lower_case=a_	,	never_split=['''[UNK]''']			)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			)	,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)




							def 			snake_case__					(       self							:			str			):
									'''simple docstring'''





									__UpperCAmelCase   :					List[Any]													=     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									__UpperCAmelCase   :					List[str]													=     {}
									for i, token in enumerate(a_			):
											__UpperCAmelCase   :					List[Any]													=     i
									__UpperCAmelCase   :					List[Any]													=     WordpieceTokenizer(vocab=a_	,	unk_token='''[UNK]'''			)

									self.assertListEqual(tokenizer.tokenize(''''''			)	,	[]			)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''			)	,	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			)	,	['''[UNK]''', '''runn''', '''##ing''']			)




							def 			snake_case__					(       self							:			List[Any]			):
									'''simple docstring'''





									self.assertTrue(_is_whitespace(''' '''			)			)
									self.assertTrue(_is_whitespace('''\t'''			)			)
									self.assertTrue(_is_whitespace('''\r'''			)			)
									self.assertTrue(_is_whitespace('''\n'''			)			)
									self.assertTrue(_is_whitespace('''\u00A0'''			)			)

									self.assertFalse(_is_whitespace('''A'''			)			)
									self.assertFalse(_is_whitespace('''-'''			)			)




							def 			snake_case__					(       self							:			int			):
									'''simple docstring'''





									self.assertTrue(_is_control('''\u0005'''			)			)

									self.assertFalse(_is_control('''A'''			)			)
									self.assertFalse(_is_control(''' '''			)			)
									self.assertFalse(_is_control('''\t'''			)			)
									self.assertFalse(_is_control('''\r'''			)			)




							def 			snake_case__					(       self							:			Tuple			):
									'''simple docstring'''





									self.assertTrue(_is_punctuation('''-'''			)			)
									self.assertTrue(_is_punctuation('''$'''			)			)
									self.assertTrue(_is_punctuation('''`'''			)			)
									self.assertTrue(_is_punctuation('''.'''			)			)

									self.assertFalse(_is_punctuation('''A'''			)			)
									self.assertFalse(_is_punctuation(''' '''			)			)




							def 			snake_case__					(       self							:			Optional[int]			):
									'''simple docstring'''





									__UpperCAmelCase   :					Any													=     self.get_tokenizer()
									__UpperCAmelCase   :					Any													=     self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(a_			) for t in ['''Test''', '''\xad''', '''test''']]	,	[['''[UNK]'''], [], ['''[UNK]''']]			)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(a_			) for t in ['''Test''', '''\xad''', '''test''']]	,	[['''[UNK]'''], [], ['''[UNK]''']]			)




							@slow
							def 			snake_case__					(       self							:			str			):
									'''simple docstring'''





									__UpperCAmelCase   :					Dict													=     self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''			)

									__UpperCAmelCase   :					Tuple													=     tokenizer.encode('''sequence builders'''	,	add_special_tokens=a_			)
									__UpperCAmelCase   :					Any													=     tokenizer.encode('''multi-sequence build'''	,	add_special_tokens=a_			)

									__UpperCAmelCase   :					Optional[Any]													=     tokenizer.build_inputs_with_special_tokens(a_			)
									__UpperCAmelCase   :					List[Any]													=     tokenizer.build_inputs_with_special_tokens(a_	,	a_			)

									assert encoded_sentence == [1_01] + text + [1_02]
									assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]




							def 			snake_case__					(       self							:			List[str]			):
									'''simple docstring'''





									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'			):
													__UpperCAmelCase   :					str													=     self.rust_tokenizer_class.from_pretrained(a_	,	**a_			)

													__UpperCAmelCase   :					Tuple													=     F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													__UpperCAmelCase   :					Tuple													=     tokenizer_r.encode_plus(
													    a_	,	return_attention_mask=a_	,	return_token_type_ids=a_	,	return_offsets_mapping=a_	,	add_special_tokens=a_	,	)

													__UpperCAmelCase   :					Any													=     tokenizer_r.do_lower_case if hasattr(a_	,	'''do_lower_case'''			) else False
													__UpperCAmelCase   :					Optional[int]													=     (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]	,	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
													self.assertEqual([e[0] for e in expected_results]	,	tokens['''offset_mapping''']			)




							def 			snake_case__					(       self							:			List[str]			):
									'''simple docstring'''





									__UpperCAmelCase   :					Union[str, Any]													=     ['''的''', '''人''', '''有''']
									__UpperCAmelCase   :					Any													=     ''''''.join(a_			)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'			):
													__UpperCAmelCase   :					Tuple													=     True
													__UpperCAmelCase   :					Any													=     self.tokenizer_class.from_pretrained(a_	,	**a_			)
													__UpperCAmelCase   :					List[Any]													=     self.rust_tokenizer_class.from_pretrained(a_	,	**a_			)

													__UpperCAmelCase   :					int													=     tokenizer_p.encode(a_	,	add_special_tokens=a_			)
													__UpperCAmelCase   :					Optional[Any]													=     tokenizer_r.encode(a_	,	add_special_tokens=a_			)

													__UpperCAmelCase   :					Tuple													=     tokenizer_r.convert_ids_to_tokens(a_			)
													__UpperCAmelCase   :					Optional[int]													=     tokenizer_p.convert_ids_to_tokens(a_			)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(a_	,	a_			)
													self.assertListEqual(a_	,	a_			)

													__UpperCAmelCase   :					str													=     False
													__UpperCAmelCase   :					Optional[int]													=     self.rust_tokenizer_class.from_pretrained(a_	,	**a_			)
													__UpperCAmelCase   :					Optional[int]													=     self.tokenizer_class.from_pretrained(a_	,	**a_			)

													__UpperCAmelCase   :					List[Any]													=     tokenizer_r.encode(a_	,	add_special_tokens=a_			)
													__UpperCAmelCase   :					List[Any]													=     tokenizer_p.encode(a_	,	add_special_tokens=a_			)

													__UpperCAmelCase   :					Optional[int]													=     tokenizer_r.convert_ids_to_tokens(a_			)
													__UpperCAmelCase   :					Any													=     tokenizer_p.convert_ids_to_tokens(a_			)

													# it is expected that only the first Chinese character is not preceded by "##".
													__UpperCAmelCase   :					Optional[Any]													=     [
													    F'##{token}' if idx != 0 else token for idx, token in enumerate(a_			)
													]
													self.assertListEqual(a_	,	a_			)
													self.assertListEqual(a_	,	a_			)










import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir

@dataclass
class 		UpperCAmelCase__      :




			'''simple docstring'''






			UpperCamelCase      						=	42

class 		UpperCAmelCase__      (      unittest.TestCase       ):




			'''simple docstring'''






			def 			snake_case__					(       self							:			Any			):
					'''simple docstring'''





					__UpperCAmelCase   :					Optional[int]													=     get_tests_dir('''fixtures/test_sentencepiece.model'''			)
					__UpperCAmelCase   :					Union[str, Any]													=     get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model'''			)

					__UpperCAmelCase   :					List[str]													=     FakeOriginalTokenizer(vocab_file=a_			)

					with warnings.catch_warnings(record=a_			) as w:
							__UpperCAmelCase   :					Optional[int]													=     SpmConverter(a_			)
					self.assertEqual(len(a_			)	,	0			)

					__UpperCAmelCase   :					Optional[int]													=     FakeOriginalTokenizer(vocab_file=a_			)

					with warnings.catch_warnings(record=a_			) as w:
							__UpperCAmelCase   :					Dict													=     SpmConverter(a_			)
					self.assertEqual(len(a_			)	,	1			)

					self.assertIn(
					    '''The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'''
					    ''' which is not implemented in the fast tokenizers.'''	,	str(w[0].message			)	,	)





