






"""simple docstring"""





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 		_a       (						lowercase_    ,					unittest.TestCase   ):





			'''simple docstring'''






			UpperCamelCase__      					=    ElectraTokenizer
			UpperCamelCase__      					=    ElectraTokenizerFast
			UpperCamelCase__      					=    True
			UpperCamelCase__      					=    True
			UpperCamelCase__      					=    filter_non_english





			def __lowercase    (   self)	-> str:




										'''simple docstring'''




										super().setUp()

										lowercase__:    List[str]        =      [
										    "[UNK]",
										    "[CLS]",
										    "[SEP]",
										    "[PAD]",
										    "[MASK]",
										    "want",
										    "##want",
										    "##ed",
										    "wa",
										    "un",
										    "runn",
										    "##ing",
										    ",",
										    "low",
										    "lowest",
										]
										lowercase__:    int        =      os.path.join(self.tmpdirname			,				VOCAB_FILES_NAMES["vocab_file"])
										with open(self.vocab_file			,				"w"			,				encoding="utf-8") as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))





			def __lowercase    (   self			,				UpperCAmelCase_)	-> Dict:




										'''simple docstring'''




										lowercase__:    str        =      "UNwant\u00E9d,running"
										lowercase__:    List[Any]        =      "unwanted, running"
										return input_text, output_text





			def __lowercase    (   self)	-> Union[str, Any]:




										'''simple docstring'''




										lowercase__:    str        =      self.tokenizer_class(self.vocab_file)

										lowercase__:    Tuple        =      tokenizer.tokenize("UNwant\u00E9d,running")
										self.assertListEqual(UpperCAmelCase_			,				["un", "##want", "##ed", ",", "runn", "##ing"])
										self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)			,				[9, 6, 7, 12, 10, 11])





			def __lowercase    (   self)	-> Optional[int]:




										'''simple docstring'''




										if not self.test_rust_tokenizer:
																	return

										lowercase__:    Dict        =      self.get_tokenizer()
										lowercase__:    List[Any]        =      self.get_rust_tokenizer()

										lowercase__:    Optional[Any]        =      "UNwant\u00E9d,running"

										lowercase__:    Tuple        =      tokenizer.tokenize(UpperCAmelCase_)
										lowercase__:    Tuple        =      rust_tokenizer.tokenize(UpperCAmelCase_)
										self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)

										lowercase__:    List[str]        =      tokenizer.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)
										lowercase__:    Optional[Any]        =      rust_tokenizer.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)
										self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)

										lowercase__:    List[Any]        =      self.get_rust_tokenizer()
										lowercase__:    int        =      tokenizer.encode(UpperCAmelCase_)
										lowercase__:    str        =      rust_tokenizer.encode(UpperCAmelCase_)
										self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)

										# With lower casing
										lowercase__:    Optional[Any]        =      self.get_tokenizer(do_lower_case=UpperCAmelCase_)
										lowercase__:    List[Any]        =      self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_)

										lowercase__:    str        =      "UNwant\u00E9d,running"

										lowercase__:    str        =      tokenizer.tokenize(UpperCAmelCase_)
										lowercase__:    Tuple        =      rust_tokenizer.tokenize(UpperCAmelCase_)
										self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)

										lowercase__:    List[Any]        =      tokenizer.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)
										lowercase__:    List[str]        =      rust_tokenizer.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)
										self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)

										lowercase__:    Any        =      self.get_rust_tokenizer()
										lowercase__:    Dict        =      tokenizer.encode(UpperCAmelCase_)
										lowercase__:    List[Any]        =      rust_tokenizer.encode(UpperCAmelCase_)
										self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)





			def __lowercase    (   self)	-> Dict:




										'''simple docstring'''




										lowercase__:    Optional[int]        =      BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")			,				["ah", "\u535A", "\u63A8", "zz"])





			def __lowercase    (   self)	-> Optional[int]:




										'''simple docstring'''




										lowercase__:    str        =      BasicTokenizer(do_lower_case=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,				["hello", "!", "how", "are", "you", "?"])
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,				["hello"])





			def __lowercase    (   self)	-> Union[str, Any]:




										'''simple docstring'''




										lowercase__:    Any        =      BasicTokenizer(do_lower_case=UpperCAmelCase_			,				strip_accents=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,				["hällo", "!", "how", "are", "you", "?"])
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,				["h\u00E9llo"])





			def __lowercase    (   self)	-> int:




										'''simple docstring'''




										lowercase__:    Tuple        =      BasicTokenizer(do_lower_case=UpperCAmelCase_			,				strip_accents=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,				["hallo", "!", "how", "are", "you", "?"])
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,				["hello"])





			def __lowercase    (   self)	-> Union[str, Any]:




										'''simple docstring'''




										lowercase__:    int        =      BasicTokenizer(do_lower_case=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,				["hallo", "!", "how", "are", "you", "?"])
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,				["hello"])





			def __lowercase    (   self)	-> Optional[Any]:




										'''simple docstring'''




										lowercase__:    Optional[int]        =      BasicTokenizer(do_lower_case=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,				["HeLLo", "!", "how", "Are", "yoU", "?"])





			def __lowercase    (   self)	-> Optional[Any]:




										'''simple docstring'''




										lowercase__:    Optional[Any]        =      BasicTokenizer(do_lower_case=UpperCAmelCase_			,				strip_accents=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,				["HäLLo", "!", "how", "Are", "yoU", "?"])





			def __lowercase    (   self)	-> Union[str, Any]:




										'''simple docstring'''




										lowercase__:    Any        =      BasicTokenizer(do_lower_case=UpperCAmelCase_			,				strip_accents=UpperCAmelCase_)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,				["HaLLo", "!", "how", "Are", "yoU", "?"])





			def __lowercase    (   self)	-> int:




										'''simple docstring'''




										lowercase__:    List[Any]        =      BasicTokenizer(do_lower_case=UpperCAmelCase_			,				never_split=["[UNK]"])

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")			,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])





			def __lowercase    (   self)	-> int:




										'''simple docstring'''




										lowercase__:    List[str]        =      ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

										lowercase__:    Dict        =      {}
										for i, token in enumerate(UpperCAmelCase_):
																	lowercase__:    Optional[int]        =      i
										lowercase__:    str        =      WordpieceTokenizer(vocab=UpperCAmelCase_			,				unk_token="[UNK]")

										self.assertListEqual(tokenizer.tokenize("")			,				[])

										self.assertListEqual(tokenizer.tokenize("unwanted running")			,				["un", "##want", "##ed", "runn", "##ing"])

										self.assertListEqual(tokenizer.tokenize("unwantedX running")			,				["[UNK]", "runn", "##ing"])





			def __lowercase    (   self)	-> Dict:




										'''simple docstring'''




										self.assertTrue(_is_whitespace(" "))
										self.assertTrue(_is_whitespace("\t"))
										self.assertTrue(_is_whitespace("\r"))
										self.assertTrue(_is_whitespace("\n"))
										self.assertTrue(_is_whitespace("\u00A0"))

										self.assertFalse(_is_whitespace("A"))
										self.assertFalse(_is_whitespace("-"))





			def __lowercase    (   self)	-> List[Any]:




										'''simple docstring'''




										self.assertTrue(_is_control("\u0005"))

										self.assertFalse(_is_control("A"))
										self.assertFalse(_is_control(" "))
										self.assertFalse(_is_control("\t"))
										self.assertFalse(_is_control("\r"))





			def __lowercase    (   self)	-> List[Any]:




										'''simple docstring'''




										self.assertTrue(_is_punctuation("-"))
										self.assertTrue(_is_punctuation("$"))
										self.assertTrue(_is_punctuation("`"))
										self.assertTrue(_is_punctuation("."))

										self.assertFalse(_is_punctuation("A"))
										self.assertFalse(_is_punctuation(" "))





			def __lowercase    (   self)	-> Tuple:




										'''simple docstring'''




										lowercase__:    Optional[Any]        =      self.get_tokenizer()
										lowercase__:    str        =      self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_) for t in ["Test", "\xad", "test"]]			,				[["[UNK]"], [], ["[UNK]"]])

										self.assertListEqual(
										    [rust_tokenizer.tokenize(UpperCAmelCase_) for t in ["Test", "\xad", "test"]]			,				[["[UNK]"], [], ["[UNK]"]])





			@slow
			def __lowercase    (   self)	-> int:




										'''simple docstring'''




										lowercase__:    Dict        =      self.tokenizer_class.from_pretrained("google/electra-base-discriminator")

										lowercase__:    Optional[int]        =      tokenizer.encode("sequence builders"			,				add_special_tokens=UpperCAmelCase_)
										lowercase__:    List[str]        =      tokenizer.encode("multi-sequence build"			,				add_special_tokens=UpperCAmelCase_)

										lowercase__:    Dict        =      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_)
										lowercase__:    Union[str, Any]        =      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_			,				UpperCAmelCase_)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]





			def __lowercase    (   self)	-> List[Any]:




										'''simple docstring'''




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																								lowercase__:    Any        =      self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_			,				**UpperCAmelCase_)

																								lowercase__:    int        =      F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																								lowercase__:    Tuple        =      tokenizer_r.encode_plus(
																								    UpperCAmelCase_			,				return_attention_mask=UpperCAmelCase_			,				return_token_type_ids=UpperCAmelCase_			,				return_offsets_mapping=UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_			,				)

																								lowercase__:    str        =      tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_			,				"do_lower_case") else False
																								lowercase__:    List[Any]        =      (
																								    [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "A"),
																								        ((1, 2), ","),
																								        ((3, 5), "na"),
																								        ((5, 6), "##ï"),
																								        ((6, 8), "##ve"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "Allen"),
																								        ((21, 23), "##NL"),
																								        ((23, 24), "##P"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								    if not do_lower_case
																								    else [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "a"),
																								        ((1, 2), ","),
																								        ((3, 8), "naive"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "allen"),
																								        ((21, 23), "##nl"),
																								        ((23, 24), "##p"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								)

																								self.assertEqual(
																								    [e[1] for e in expected_results]			,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
																								self.assertEqual([e[0] for e in expected_results]			,				tokens["offset_mapping"])





			def __lowercase    (   self)	-> List[Any]:




										'''simple docstring'''




										lowercase__:    Union[str, Any]        =      ["的", "人", "有"]
										lowercase__:    Any        =      "".join(UpperCAmelCase_)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																								lowercase__:    Optional[Any]        =      True
																								lowercase__:    List[str]        =      self.tokenizer_class.from_pretrained(UpperCAmelCase_			,				**UpperCAmelCase_)
																								lowercase__:    Dict        =      self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_			,				**UpperCAmelCase_)

																								lowercase__:    str        =      tokenizer_p.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)
																								lowercase__:    Optional[int]        =      tokenizer_r.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)

																								lowercase__:    Union[str, Any]        =      tokenizer_r.convert_ids_to_tokens(UpperCAmelCase_)
																								lowercase__:    List[str]        =      tokenizer_p.convert_ids_to_tokens(UpperCAmelCase_)

																								# it is expected that each Chinese character is not preceded by "##"
																								self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)
																								self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)

																								lowercase__:    Dict        =      False
																								lowercase__:    Optional[int]        =      self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_			,				**UpperCAmelCase_)
																								lowercase__:    int        =      self.tokenizer_class.from_pretrained(UpperCAmelCase_			,				**UpperCAmelCase_)

																								lowercase__:    str        =      tokenizer_r.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)
																								lowercase__:    Tuple        =      tokenizer_p.encode(UpperCAmelCase_			,				add_special_tokens=UpperCAmelCase_)

																								lowercase__:    Union[str, Any]        =      tokenizer_r.convert_ids_to_tokens(UpperCAmelCase_)
																								lowercase__:    List[str]        =      tokenizer_p.convert_ids_to_tokens(UpperCAmelCase_)

																								# it is expected that only the first Chinese character is not preceded by "##".
																								lowercase__:    List[str]        =      [
																								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase_)
																								]
																								self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)
																								self.assertListEqual(UpperCAmelCase_			,				UpperCAmelCase_)














"""simple docstring"""





def     A(  snake_case_						,       snake_case_		):
       """simple docstring"""





       assert isinstance(snake_case_						,       snake_case_		), F"""The input value of [set_a={set_a}] is not a set"""
       assert isinstance(snake_case_						,       snake_case_		), F"""The input value of [set_b={set_b}] is not a set"""

       return {a + b for a in set_a for b in set_b}


if __name__ == "__main__":
  from doctest import testmod

  testmod()






