



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 	__snake_case (				SCREAMING_SNAKE_CASE     ,unittest.TestCase):







							'''simple docstring'''






							UpperCamelCase__			: int             =   ElectraTokenizer
							UpperCamelCase__			: Optional[int]             =   ElectraTokenizerFast
							UpperCamelCase__			: int             =   True
							UpperCamelCase__			: int             =   True
							UpperCamelCase__			: List[str]             =   filter_non_english


							def _a (							self						):
											super().setUp()

											a__			 =						[
											    """[UNK]""",
											    """[CLS]""",
											    """[SEP]""",
											    """[PAD]""",
											    """[MASK]""",
											    """want""",
											    """##want""",
											    """##ed""",
											    """wa""",
											    """un""",
											    """runn""",
											    """##ing""",
											    """,""",
											    """low""",
											    """lowest""",
											]
											a__			 =						os.path.join(self.tmpdirname					,					VOCAB_FILES_NAMES["""vocab_file"""]						)
											with open(self.vocab_file					,					"""w"""					,					encoding="""utf-8"""						) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)


							def _a (							self					,					a_						):
											a__			 =						"""UNwant\u00E9d,running"""
											a__			 =						"""unwanted, running"""
											return input_text, output_text


							def _a (							self						):
											a__			 =						self.tokenizer_class(self.vocab_file						)

											a__			 =						tokenizer.tokenize("""UNwant\u00E9d,running"""						)
											self.assertListEqual(a_					,					["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]						)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(a_						)					,					[9, 6, 7, 12, 10, 11]						)


							def _a (							self						):
											if not self.test_rust_tokenizer:
															return

											a__			 =						self.get_tokenizer()
											a__			 =						self.get_rust_tokenizer()

											a__			 =						"""UNwant\u00E9d,running"""

											a__			 =						tokenizer.tokenize(a_						)
											a__			 =						rust_tokenizer.tokenize(a_						)
											self.assertListEqual(a_					,					a_						)

											a__			 =						tokenizer.encode(a_					,					add_special_tokens=a_						)
											a__			 =						rust_tokenizer.encode(a_					,					add_special_tokens=a_						)
											self.assertListEqual(a_					,					a_						)

											a__			 =						self.get_rust_tokenizer()
											a__			 =						tokenizer.encode(a_						)
											a__			 =						rust_tokenizer.encode(a_						)
											self.assertListEqual(a_					,					a_						)

											# With lower casing
											a__			 =						self.get_tokenizer(do_lower_case=a_						)
											a__			 =						self.get_rust_tokenizer(do_lower_case=a_						)

											a__			 =						"""UNwant\u00E9d,running"""

											a__			 =						tokenizer.tokenize(a_						)
											a__			 =						rust_tokenizer.tokenize(a_						)
											self.assertListEqual(a_					,					a_						)

											a__			 =						tokenizer.encode(a_					,					add_special_tokens=a_						)
											a__			 =						rust_tokenizer.encode(a_					,					add_special_tokens=a_						)
											self.assertListEqual(a_					,					a_						)

											a__			 =						self.get_rust_tokenizer()
											a__			 =						tokenizer.encode(a_						)
											a__			 =						rust_tokenizer.encode(a_						)
											self.assertListEqual(a_					,					a_						)


							def _a (							self						):
											a__			 =						BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""						)					,					["""ah""", """\u535A""", """\u63A8""", """zz"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)					,					["""hello""", """!""", """how""", """are""", """you""", """?"""]						)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)					,					["""hello"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_					,					strip_accents=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)					,					["""hällo""", """!""", """how""", """are""", """you""", """?"""]						)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)					,					["""h\u00E9llo"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_					,					strip_accents=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)					,					["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)					,					["""hello"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)					,					["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)					,					["""hello"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)					,					["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_					,					strip_accents=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)					,					["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_					,					strip_accents=a_						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)					,					["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)


							def _a (							self						):
											a__			 =						BasicTokenizer(do_lower_case=a_					,					never_split=["""[UNK]"""]						)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""						)					,					["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]						)


							def _a (							self						):
											a__			 =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

											a__			 =						{}
											for i, token in enumerate(a_						):
															a__			 =						i
											a__			 =						WordpieceTokenizer(vocab=a_					,					unk_token="""[UNK]"""						)

											self.assertListEqual(tokenizer.tokenize(""""""						)					,					[]						)

											self.assertListEqual(tokenizer.tokenize("""unwanted running"""						)					,					["""un""", """##want""", """##ed""", """runn""", """##ing"""]						)

											self.assertListEqual(tokenizer.tokenize("""unwantedX running"""						)					,					["""[UNK]""", """runn""", """##ing"""]						)


							def _a (							self						):
											self.assertTrue(_is_whitespace(""" """						)						)
											self.assertTrue(_is_whitespace("""\t"""						)						)
											self.assertTrue(_is_whitespace("""\r"""						)						)
											self.assertTrue(_is_whitespace("""\n"""						)						)
											self.assertTrue(_is_whitespace("""\u00A0"""						)						)

											self.assertFalse(_is_whitespace("""A"""						)						)
											self.assertFalse(_is_whitespace("""-"""						)						)


							def _a (							self						):
											self.assertTrue(_is_control("""\u0005"""						)						)

											self.assertFalse(_is_control("""A"""						)						)
											self.assertFalse(_is_control(""" """						)						)
											self.assertFalse(_is_control("""\t"""						)						)
											self.assertFalse(_is_control("""\r"""						)						)


							def _a (							self						):
											self.assertTrue(_is_punctuation("""-"""						)						)
											self.assertTrue(_is_punctuation("""$"""						)						)
											self.assertTrue(_is_punctuation("""`"""						)						)
											self.assertTrue(_is_punctuation("""."""						)						)

											self.assertFalse(_is_punctuation("""A"""						)						)
											self.assertFalse(_is_punctuation(""" """						)						)


							def _a (							self						):
											a__			 =						self.get_tokenizer()
											a__			 =						self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(a_						) for t in ["""Test""", """\xad""", """test"""]]					,					[["""[UNK]"""], [], ["""[UNK]"""]]						)

											self.assertListEqual(
											    [rust_tokenizer.tokenize(a_						) for t in ["""Test""", """\xad""", """test"""]]					,					[["""[UNK]"""], [], ["""[UNK]"""]]						)


							@slow
							def _a (							self						):
											a__			 =						self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""						)

											a__			 =						tokenizer.encode("""sequence builders"""					,					add_special_tokens=a_						)
											a__			 =						tokenizer.encode("""multi-sequence build"""					,					add_special_tokens=a_						)

											a__			 =						tokenizer.build_inputs_with_special_tokens(a_						)
											a__			 =						tokenizer.build_inputs_with_special_tokens(a_					,					a_						)

											assert encoded_sentence == [101] + text + [102]
											assert encoded_pair == [101] + text + [102] + text_a + [102]


							def _a (							self						):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
																			a__			 =						self.rust_tokenizer_class.from_pretrained(a_					,					**a_						)

																			a__			 =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			a__			 =						tokenizer_r.encode_plus(
																			    a_					,					return_attention_mask=a_					,					return_token_type_ids=a_					,					return_offsets_mapping=a_					,					add_special_tokens=a_					,					)

																			a__			 =						tokenizer_r.do_lower_case if hasattr(a_					,					"""do_lower_case"""						) else False
																			a__			 =						(
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), """A"""),
																			        ((1, 2), ""","""),
																			        ((3, 5), """na"""),
																			        ((5, 6), """##ï"""),
																			        ((6, 8), """##ve"""),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), """Allen"""),
																			        ((21, 23), """##NL"""),
																			        ((23, 24), """##P"""),
																			        ((25, 33), """sentence"""),
																			        ((33, 34), """."""),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), """a"""),
																			        ((1, 2), ""","""),
																			        ((3, 8), """naive"""),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), """allen"""),
																			        ((21, 23), """##nl"""),
																			        ((23, 24), """##p"""),
																			        ((25, 33), """sentence"""),
																			        ((33, 34), """."""),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]					,					tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]						)						)
																			self.assertEqual([e[0] for e in expected_results]					,					tokens["""offset_mapping"""]						)







							def _a (							self						):
											a__			 =						["""的""", """人""", """有"""]
											a__			 =						"""""".join(a_						)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
																			a__			 =						True
																			a__			 =						self.tokenizer_class.from_pretrained(a_					,					**a_						)
																			a__			 =						self.rust_tokenizer_class.from_pretrained(a_					,					**a_						)

																			a__			 =						tokenizer_p.encode(a_					,					add_special_tokens=a_						)
																			a__			 =						tokenizer_r.encode(a_					,					add_special_tokens=a_						)

																			a__			 =						tokenizer_r.convert_ids_to_tokens(a_						)
																			a__			 =						tokenizer_p.convert_ids_to_tokens(a_						)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(a_					,					a_						)
																			self.assertListEqual(a_					,					a_						)

																			a__			 =						False
																			a__			 =						self.rust_tokenizer_class.from_pretrained(a_					,					**a_						)
																			a__			 =						self.tokenizer_class.from_pretrained(a_					,					**a_						)

																			a__			 =						tokenizer_r.encode(a_					,					add_special_tokens=a_						)
																			a__			 =						tokenizer_p.encode(a_					,					add_special_tokens=a_						)

																			a__			 =						tokenizer_r.convert_ids_to_tokens(a_						)
																			a__			 =						tokenizer_p.convert_ids_to_tokens(a_						)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			a__			 =						[
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(a_						)
																			]
																			self.assertListEqual(a_					,					a_						)
																			self.assertListEqual(a_					,					a_						)






import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
	from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

	from .test_feature_extraction_speechta import floats_list


UpperCAmelCase								=   get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")

@require_torch
class 	__snake_case (				unittest.TestCase):







							'''simple docstring'''






							def _a (							self						):
											a__			 =						tempfile.mkdtemp()

											a__			 =						SpeechTaTokenizer(a_						)
											tokenizer.save_pretrained(self.tmpdirname						)

											a__			 =						{
											    """feature_size""": 1,
											    """padding_value""": 0.0,
											    """sampling_rate""": 16_000,
											    """do_normalize""": False,
											    """num_mel_bins""": 80,
											    """hop_length""": 16,
											    """win_length""": 64,
											    """win_function""": """hann_window""",
											    """fmin""": 80,
											    """fmax""": 7_600,
											    """mel_floor""": 1E-1_0,
											    """reduction_factor""": 2,
											    """return_attention_mask""": True,
											}

											a__			 =						os.path.join(self.tmpdirname					,					a_						)
											with open(self.feature_extraction_file					,					"""w"""					,					encoding="""utf-8"""						) as fp:
															fp.write(json.dumps(a_						) + """\n"""						)


							def _a (							self					,					**a_						):
											return SpeechTaTokenizer.from_pretrained(self.tmpdirname					,					**a_						)


							def _a (							self					,					**a_						):
											return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname					,					**a_						)


							def _a (							self						):
											shutil.rmtree(self.tmpdirname						)


							def _a (							self						):
											a__			 =						self.get_tokenizer()
											a__			 =						self.get_feature_extractor()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											processor.save_pretrained(self.tmpdirname						)
											a__			 =						SpeechTaProcessor.from_pretrained(self.tmpdirname						)

											self.assertEqual(processor.tokenizer.get_vocab()					,					tokenizer.get_vocab()						)
											self.assertIsInstance(processor.tokenizer					,					a_						)

											self.assertEqual(processor.feature_extractor.to_json_string()					,					feature_extractor.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor					,					a_						)


							def _a (							self						):
											a__			 =						SpeechTaProcessor(tokenizer=self.get_tokenizer()					,					feature_extractor=self.get_feature_extractor()						)
											processor.save_pretrained(self.tmpdirname						)

											a__			 =						self.get_tokenizer(bos_token="""(BOS)"""					,					eos_token="""(EOS)"""						)
											a__			 =						self.get_feature_extractor(do_normalize=a_					,					padding_value=1.0						)

											a__			 =						SpeechTaProcessor.from_pretrained(
											    self.tmpdirname					,					bos_token="""(BOS)"""					,					eos_token="""(EOS)"""					,					do_normalize=a_					,					padding_value=1.0						)

											self.assertEqual(processor.tokenizer.get_vocab()					,					tokenizer_add_kwargs.get_vocab()						)
											self.assertIsInstance(processor.tokenizer					,					a_						)

											self.assertEqual(processor.feature_extractor.to_json_string()					,					feature_extractor_add_kwargs.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor					,					a_						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						floats_list((3, 1_000)						)

											a__			 =						feature_extractor(audio=a_					,					return_tensors="""np"""						)
											a__			 =						processor(audio=a_					,					return_tensors="""np"""						)

											for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()					,					input_processor[key].sum()					,					delta=1E-2						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						floats_list((3, 1_000)						)

											a__			 =						feature_extractor(audio_target=a_					,					return_tensors="""np"""						)
											a__			 =						processor(audio_target=a_					,					return_tensors="""np"""						)

											for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()					,					input_processor[key].sum()					,					delta=1E-2						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						"""This is a test string"""

											a__			 =						processor(text=a_						)
											a__			 =						tokenizer(a_						)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]					,					encoded_processor[key]						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						"""This is a test string"""

											a__			 =						processor(text_target=a_						)
											a__			 =						tokenizer(a_						)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]					,					encoded_processor[key]						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											a__			 =						processor.batch_decode(a_						)
											a__			 =						tokenizer.batch_decode(a_						)

											self.assertListEqual(a_					,					a_						)







							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											self.assertListEqual(
											    processor.model_input_names					,					feature_extractor.model_input_names					,					msg="""`processor` and `feature_extractor` model input names do not match"""					,					)

