
'''simple docstring'''



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class      A							(    __UpperCAmelCase						,  unittest.TestCase			):
			lowerCamelCase							:							int    						=					ElectraTokenizer
			lowerCamelCase							:							Dict    						=					ElectraTokenizerFast
			lowerCamelCase							:							str    						=					True
			lowerCamelCase							:							Union[str, Any]    						=					True
			lowerCamelCase							:							int    						=					filter_non_english



			def A__	(   self    )     ->						List[Any]:




									'''simple docstring'''



									super().setUp()

									lowercase__								=    [
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									lowercase__								=    os.path.join(self.tmpdirname    ,       VOCAB_FILES_NAMES["""vocab_file"""]    )
									with open(self.vocab_file    ,       """w"""    ,       encoding="""utf-8"""    ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )



			def A__	(   self    ,       lowerCamelCase__    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    """UNwant\u00E9d,running"""
									lowercase__								=    """unwanted, running"""
									return input_text, output_text



			def A__	(   self    )     ->						Dict:




									'''simple docstring'''



									lowercase__								=    self.tokenizer_class(self.vocab_file    )

									lowercase__								=    tokenizer.tokenize("""UNwant\u00E9d,running"""    )
									self.assertListEqual(lowerCamelCase__    ,       ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]    )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__    )    ,       [9, 6, 7, 12, 10, 11]    )



			def A__	(   self    )     ->						List[str]:




									'''simple docstring'''



									if not self.test_rust_tokenizer:
															return

									lowercase__								=    self.get_tokenizer()
									lowercase__								=    self.get_rust_tokenizer()

									lowercase__								=    """UNwant\u00E9d,running"""

									lowercase__								=    tokenizer.tokenize(lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.tokenize(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									lowercase__								=    tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									lowercase__								=    self.get_rust_tokenizer()
									lowercase__								=    tokenizer.encode(lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.encode(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									# With lower casing
									lowercase__								=    self.get_tokenizer(do_lower_case=lowerCamelCase__    )
									lowercase__								=    self.get_rust_tokenizer(do_lower_case=lowerCamelCase__    )

									lowercase__								=    """UNwant\u00E9d,running"""

									lowercase__								=    tokenizer.tokenize(lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.tokenize(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									lowercase__								=    tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

									lowercase__								=    self.get_rust_tokenizer()
									lowercase__								=    tokenizer.encode(lowerCamelCase__    )
									lowercase__								=    rust_tokenizer.encode(lowerCamelCase__    )
									self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )



			def A__	(   self    )     ->						Optional[int]:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )    ,       ["""ah""", """\u535A""", """\u63A8""", """zz"""]    )



			def A__	(   self    )     ->						int:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )    ,       ["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )    ,       ["""hello"""]    )



			def A__	(   self    )     ->						str:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    ,       strip_accents=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )    ,       ["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )    ,       ["""h\u00E9llo"""]    )



			def A__	(   self    )     ->						Optional[Any]:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    ,       strip_accents=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )    ,       ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )    ,       ["""hello"""]    )



			def A__	(   self    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )    ,       ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )    ,       ["""hello"""]    )



			def A__	(   self    )     ->						Dict:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )    ,       ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )



			def A__	(   self    )     ->						List[Any]:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    ,       strip_accents=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )    ,       ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )



			def A__	(   self    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    ,       strip_accents=lowerCamelCase__    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )    ,       ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )



			def A__	(   self    )     ->						Tuple:




									'''simple docstring'''



									lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    ,       never_split=["""[UNK]"""]    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )    ,       ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )



			def A__	(   self    )     ->						Dict:




									'''simple docstring'''



									lowercase__								=    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									lowercase__								=    {}
									for i, token in enumerate(lowerCamelCase__    ):
															lowercase__								=    i
									lowercase__								=    WordpieceTokenizer(vocab=lowerCamelCase__    ,       unk_token="""[UNK]"""    )

									self.assertListEqual(tokenizer.tokenize(""""""    )    ,       []    )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )    ,       ["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )    ,       ["""[UNK]""", """runn""", """##ing"""]    )



			def A__	(   self    )     ->						Any:




									'''simple docstring'''



									self.assertTrue(_is_whitespace(""" """    )    )
									self.assertTrue(_is_whitespace("""\t"""    )    )
									self.assertTrue(_is_whitespace("""\r"""    )    )
									self.assertTrue(_is_whitespace("""\n"""    )    )
									self.assertTrue(_is_whitespace("""\u00A0"""    )    )

									self.assertFalse(_is_whitespace("""A"""    )    )
									self.assertFalse(_is_whitespace("""-"""    )    )



			def A__	(   self    )     ->						int:




									'''simple docstring'''



									self.assertTrue(_is_control("""\u0005"""    )    )

									self.assertFalse(_is_control("""A"""    )    )
									self.assertFalse(_is_control(""" """    )    )
									self.assertFalse(_is_control("""\t"""    )    )
									self.assertFalse(_is_control("""\r"""    )    )



			def A__	(   self    )     ->						List[Any]:




									'''simple docstring'''



									self.assertTrue(_is_punctuation("""-"""    )    )
									self.assertTrue(_is_punctuation("""$"""    )    )
									self.assertTrue(_is_punctuation("""`"""    )    )
									self.assertTrue(_is_punctuation("""."""    )    )

									self.assertFalse(_is_punctuation("""A"""    )    )
									self.assertFalse(_is_punctuation(""" """    )    )



			def A__	(   self    )     ->						Optional[Any]:




									'''simple docstring'''



									lowercase__								=    self.get_tokenizer()
									lowercase__								=    self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(lowerCamelCase__    ) for t in ["""Test""", """\xad""", """test"""]]    ,       [["""[UNK]"""], [], ["""[UNK]"""]]    )

									self.assertListEqual(
									    [rust_tokenizer.tokenize(lowerCamelCase__    ) for t in ["""Test""", """\xad""", """test"""]]    ,       [["""[UNK]"""], [], ["""[UNK]"""]]    )



			@slow
			def A__	(   self    )     ->						Any:




									'''simple docstring'''



									lowercase__								=    self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""    )

									lowercase__								=    tokenizer.encode("""sequence builders"""    ,       add_special_tokens=lowerCamelCase__    )
									lowercase__								=    tokenizer.encode("""multi-sequence build"""    ,       add_special_tokens=lowerCamelCase__    )

									lowercase__								=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    )
									lowercase__								=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase__    ,       lowerCamelCase__    )

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]



			def A__	(   self    )     ->						Dict:




									'''simple docstring'''



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																					lowercase__								=    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__    ,       **lowerCamelCase__    )

																					lowercase__								=    F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																					lowercase__								=    tokenizer_r.encode_plus(
																					    lowerCamelCase__    ,       return_attention_mask=lowerCamelCase__    ,       return_token_type_ids=lowerCamelCase__    ,       return_offsets_mapping=lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    ,       )

																					lowercase__								=    tokenizer_r.do_lower_case if hasattr(lowerCamelCase__    ,       """do_lower_case"""    ) else False
																					lowercase__								=    (
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), """A"""),
																					        ((1, 2), ""","""),
																					        ((3, 5), """na"""),
																					        ((5, 6), """##ï"""),
																					        ((6, 8), """##ve"""),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), """Allen"""),
																					        ((21, 23), """##NL"""),
																					        ((23, 24), """##P"""),
																					        ((25, 33), """sentence"""),
																					        ((33, 34), """."""),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), """a"""),
																					        ((1, 2), ""","""),
																					        ((3, 8), """naive"""),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), """allen"""),
																					        ((21, 23), """##nl"""),
																					        ((23, 24), """##p"""),
																					        ((25, 33), """sentence"""),
																					        ((33, 34), """."""),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]    ,       tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
																					self.assertEqual([e[0] for e in expected_results]    ,       tokens["""offset_mapping"""]    )



			def A__	(   self    )     ->						Optional[Any]:




									'''simple docstring'''



									lowercase__								=    ["""的""", """人""", """有"""]
									lowercase__								=    """""".join(lowerCamelCase__    )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																					lowercase__								=    True
																					lowercase__								=    self.tokenizer_class.from_pretrained(lowerCamelCase__    ,       **lowerCamelCase__    )
																					lowercase__								=    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__    ,       **lowerCamelCase__    )

																					lowercase__								=    tokenizer_p.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
																					lowercase__								=    tokenizer_r.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )

																					lowercase__								=    tokenizer_r.convert_ids_to_tokens(lowerCamelCase__    )
																					lowercase__								=    tokenizer_p.convert_ids_to_tokens(lowerCamelCase__    )

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )
																					self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )

																					lowercase__								=    False
																					lowercase__								=    self.rust_tokenizer_class.from_pretrained(lowerCamelCase__    ,       **lowerCamelCase__    )
																					lowercase__								=    self.tokenizer_class.from_pretrained(lowerCamelCase__    ,       **lowerCamelCase__    )

																					lowercase__								=    tokenizer_r.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )
																					lowercase__								=    tokenizer_p.encode(lowerCamelCase__    ,       add_special_tokens=lowerCamelCase__    )

																					lowercase__								=    tokenizer_r.convert_ids_to_tokens(lowerCamelCase__    )
																					lowercase__								=    tokenizer_p.convert_ids_to_tokens(lowerCamelCase__    )

																					# it is expected that only the first Chinese character is not preceded by "##".
																					lowercase__								=    [
																					    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase__    )
																					]
																					self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )
																					self.assertListEqual(lowerCamelCase__    ,       lowerCamelCase__    )








'''simple docstring'''



def 	_A						(							lowercase__		):
						if not isinstance(lowercase__						,						lowercase__		):
												raise ValueError("""Input series is not valid, valid series - [1, 2/3, 2]"""		)
						if len(lowercase__		) == 0:
												raise ValueError("""Input list must be a non empty list"""		)
						if len(lowercase__		) == 1 and series[0] != 0:
												return True
						lowercase__								=    []
						lowercase__								=    len(lowercase__		)
						for i in range(0						,						lowercase__		):
												if series[i] == 0:
																		raise ValueError("""Input series cannot have 0 as an element"""		)
												rec_series.append(1 / series[i]		)
						lowercase__								=    rec_series[1] - rec_series[0]
						for index in range(2						,						lowercase__		):
												if rec_series[index] - rec_series[index - 1] != common_diff:
																		return False
						return True





def 	_A						(							lowercase__		):
						if not isinstance(lowercase__						,						lowercase__		):
												raise ValueError("""Input series is not valid, valid series - [2, 4, 6]"""		)
						if len(lowercase__		) == 0:
												raise ValueError("""Input list must be a non empty list"""		)
						lowercase__								=    0
						for val in series:
												answer += 1 / val
						return len(lowercase__		) / answer


if __name__ == "__main__":
				import doctest

				doctest.testmod()






