'''simple docstring'''




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      lowerCamelCase_     (snake_case__							,			unittest.TestCase   ):
      '''simple docstring'''

      __UpperCamelCase:  Optional[int]      =						ElectraTokenizer
      __UpperCamelCase:  Optional[Any]      =						ElectraTokenizerFast
      __UpperCamelCase:  List[Any]      =						True
      __UpperCamelCase:  int      =						True
      __UpperCamelCase:  str      =						filter_non_english




      def 					_A     (	self						:    Any	):
             super().setUp()

             _UpperCAmelCase					:  str											=   [
                 "[UNK]",
                 "[CLS]",
                 "[SEP]",
                 "[PAD]",
                 "[MASK]",
                 "want",
                 "##want",
                 "##ed",
                 "wa",
                 "un",
                 "runn",
                 "##ing",
                 ",",
                 "low",
                 "lowest",
             ]
             _UpperCAmelCase					:  Optional[Any]											=   os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["vocab_file"]	)
             with open(self.vocab_file						, "w"						, encoding="utf-8"	) as vocab_writer:
                    vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)




      def 					_A     (	self						:    Optional[int]						, A						:    List[Any]	):
             _UpperCAmelCase					:  int											=   "UNwant\u00E9d,running"
             _UpperCAmelCase					:  str											=   "unwanted, running"
             return input_text, output_text




      def 					_A     (	self						:    Tuple	):
             _UpperCAmelCase					:  List[str]											=   self.tokenizer_class(self.vocab_file	)

             _UpperCAmelCase					:  List[Any]											=   tokenizer.tokenize("UNwant\u00E9d,running"	)
             self.assertListEqual(A						, ["un", "##want", "##ed", ",", "runn", "##ing"]	)
             self.assertListEqual(tokenizer.convert_tokens_to_ids(A	)						, [9, 6, 7, 12, 10, 11]	)




      def 					_A     (	self						:    Any	):
             if not self.test_rust_tokenizer:
                    return

             _UpperCAmelCase					:  Union[str, Any]											=   self.get_tokenizer()
             _UpperCAmelCase					:  List[Any]											=   self.get_rust_tokenizer()

             _UpperCAmelCase					:  Any											=   "UNwant\u00E9d,running"

             _UpperCAmelCase					:  Optional[int]											=   tokenizer.tokenize(A	)
             _UpperCAmelCase					:  List[Any]											=   rust_tokenizer.tokenize(A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode(A						, add_special_tokens=A	)
             _UpperCAmelCase					:  Optional[int]											=   rust_tokenizer.encode(A						, add_special_tokens=A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  int											=   self.get_rust_tokenizer()
             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode(A	)
             _UpperCAmelCase					:  List[str]											=   rust_tokenizer.encode(A	)
             self.assertListEqual(A						, A	)

             # With lower casing
             _UpperCAmelCase					:  Dict											=   self.get_tokenizer(do_lower_case=A	)
             _UpperCAmelCase					:  Dict											=   self.get_rust_tokenizer(do_lower_case=A	)

             _UpperCAmelCase					:  List[Any]											=   "UNwant\u00E9d,running"

             _UpperCAmelCase					:  List[Any]											=   tokenizer.tokenize(A	)
             _UpperCAmelCase					:  Dict											=   rust_tokenizer.tokenize(A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  Union[str, Any]											=   tokenizer.encode(A						, add_special_tokens=A	)
             _UpperCAmelCase					:  Optional[Any]											=   rust_tokenizer.encode(A						, add_special_tokens=A	)
             self.assertListEqual(A						, A	)

             _UpperCAmelCase					:  Dict											=   self.get_rust_tokenizer()
             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode(A	)
             _UpperCAmelCase					:  Any											=   rust_tokenizer.encode(A	)
             self.assertListEqual(A						, A	)




      def 					_A     (	self						:    Dict	):
             _UpperCAmelCase					:  Tuple											=   BasicTokenizer()

             self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)						, ["ah", "\u535A", "\u63A8", "zz"]	)




      def 					_A     (	self						:    Dict	):
             _UpperCAmelCase					:  List[Any]											=   BasicTokenizer(do_lower_case=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)						, ["hello", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["hello"]	)




      def 					_A     (	self						:    str	):
             _UpperCAmelCase					:  str											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["hällo", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["h\u00E9llo"]	)




      def 					_A     (	self						:    List[str]	):
             _UpperCAmelCase					:  Any											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["hallo", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["hello"]	)




      def 					_A     (	self						:    int	):
             _UpperCAmelCase					:  Optional[int]											=   BasicTokenizer(do_lower_case=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["hallo", "!", "how", "are", "you", "?"]	)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)						, ["hello"]	)




      def 					_A     (	self						:    int	):
             _UpperCAmelCase					:  Union[str, Any]											=   BasicTokenizer(do_lower_case=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)						, ["HeLLo", "!", "how", "Are", "yoU", "?"]	)




      def 					_A     (	self						:    Any	):
             _UpperCAmelCase					:  int											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["HäLLo", "!", "how", "Are", "yoU", "?"]	)




      def 					_A     (	self						:    Optional[int]	):
             _UpperCAmelCase					:  List[str]											=   BasicTokenizer(do_lower_case=A						, strip_accents=A	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)						, ["HaLLo", "!", "how", "Are", "yoU", "?"]	)




      def 					_A     (	self						:    Dict	):
             _UpperCAmelCase					:  Any											=   BasicTokenizer(do_lower_case=A						, never_split=["[UNK]"]	)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)						, ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)




      def 					_A     (	self						:    Optional[Any]	):
             _UpperCAmelCase					:  List[str]											=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

             _UpperCAmelCase					:  Union[str, Any]											=   {}
             for i, token in enumerate(A	):
                    _UpperCAmelCase					:  str											=   i
             _UpperCAmelCase					:  Tuple											=   WordpieceTokenizer(vocab=A						, unk_token="[UNK]"	)

             self.assertListEqual(tokenizer.tokenize(""	)						, []	)

             self.assertListEqual(tokenizer.tokenize("unwanted running"	)						, ["un", "##want", "##ed", "runn", "##ing"]	)

             self.assertListEqual(tokenizer.tokenize("unwantedX running"	)						, ["[UNK]", "runn", "##ing"]	)




      def 					_A     (	self						:    int	):
             self.assertTrue(_is_whitespace(" "	)	)
             self.assertTrue(_is_whitespace("\t"	)	)
             self.assertTrue(_is_whitespace("\r"	)	)
             self.assertTrue(_is_whitespace("\n"	)	)
             self.assertTrue(_is_whitespace("\u00A0"	)	)

             self.assertFalse(_is_whitespace("A"	)	)
             self.assertFalse(_is_whitespace("-"	)	)




      def 					_A     (	self						:    List[Any]	):
             self.assertTrue(_is_control("\u0005"	)	)

             self.assertFalse(_is_control("A"	)	)
             self.assertFalse(_is_control(" "	)	)
             self.assertFalse(_is_control("\t"	)	)
             self.assertFalse(_is_control("\r"	)	)




      def 					_A     (	self						:    Optional[Any]	):
             self.assertTrue(_is_punctuation("-"	)	)
             self.assertTrue(_is_punctuation("$"	)	)
             self.assertTrue(_is_punctuation("`"	)	)
             self.assertTrue(_is_punctuation("."	)	)

             self.assertFalse(_is_punctuation("A"	)	)
             self.assertFalse(_is_punctuation(" "	)	)




      def 					_A     (	self						:    str	):
             _UpperCAmelCase					:  Any											=   self.get_tokenizer()
             _UpperCAmelCase					:  str											=   self.get_rust_tokenizer()

             # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
             self.assertListEqual([tokenizer.tokenize(A	) for t in ["Test", "\xad", "test"]]						, [["[UNK]"], [], ["[UNK]"]]	)

             self.assertListEqual(
                 [rust_tokenizer.tokenize(A	) for t in ["Test", "\xad", "test"]]						, [["[UNK]"], [], ["[UNK]"]]	)




      @slow
      def 					_A     (	self						:    Tuple	):
             _UpperCAmelCase					:  List[str]											=   self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)

             _UpperCAmelCase					:  List[Any]											=   tokenizer.encode("sequence builders"						, add_special_tokens=A	)
             _UpperCAmelCase					:  Optional[Any]											=   tokenizer.encode("multi-sequence build"						, add_special_tokens=A	)

             _UpperCAmelCase					:  Union[str, Any]											=   tokenizer.build_inputs_with_special_tokens(A	)
             _UpperCAmelCase					:  Union[str, Any]											=   tokenizer.build_inputs_with_special_tokens(A						, A	)

             assert encoded_sentence == [101] + text + [102]
             assert encoded_pair == [101] + text + [102] + text_a + [102]




      def 					_A     (	self						:    Optional[Any]	):
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                           _UpperCAmelCase					:  Any											=   self.rust_tokenizer_class.from_pretrained(A						, **A	)

                           _UpperCAmelCase					:  Optional[int]											=   F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                           _UpperCAmelCase					:  str											=   tokenizer_r.encode_plus(
                               A						, return_attention_mask=A						, return_token_type_ids=A						, return_offsets_mapping=A						, add_special_tokens=A						, )

                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_r.do_lower_case if hasattr(A						, "do_lower_case"	) else False
                           _UpperCAmelCase					:  str											=   (
                               [
                                   ((0, 0), tokenizer_r.cls_token),
                                   ((0, 1), "A"),
                                   ((1, 2), ","),
                                   ((3, 5), "na"),
                                   ((5, 6), "##ï"),
                                   ((6, 8), "##ve"),
                                   ((9, 15), tokenizer_r.mask_token),
                                   ((16, 21), "Allen"),
                                   ((21, 23), "##NL"),
                                   ((23, 24), "##P"),
                                   ((25, 33), "sentence"),
                                   ((33, 34), "."),
                                   ((0, 0), tokenizer_r.sep_token),
                               ]
                               if not do_lower_case
                               else [
                                   ((0, 0), tokenizer_r.cls_token),
                                   ((0, 1), "a"),
                                   ((1, 2), ","),
                                   ((3, 8), "naive"),
                                   ((9, 15), tokenizer_r.mask_token),
                                   ((16, 21), "allen"),
                                   ((21, 23), "##nl"),
                                   ((23, 24), "##p"),
                                   ((25, 33), "sentence"),
                                   ((33, 34), "."),
                                   ((0, 0), tokenizer_r.sep_token),
                               ]
                           )

                           self.assertEqual(
                               [e[1] for e in expected_results]						, tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
                           self.assertEqual([e[0] for e in expected_results]						, tokens["offset_mapping"]	)





      def 					_A     (	self						:    str	):
             _UpperCAmelCase					:  int											=   ["的", "人", "有"]
             _UpperCAmelCase					:  Dict											=   "".join(A	)
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                           _UpperCAmelCase					:  Optional[int]											=   True
                           _UpperCAmelCase					:  Tuple											=   self.tokenizer_class.from_pretrained(A						, **A	)
                           _UpperCAmelCase					:  Any											=   self.rust_tokenizer_class.from_pretrained(A						, **A	)

                           _UpperCAmelCase					:  Optional[Any]											=   tokenizer_p.encode(A						, add_special_tokens=A	)
                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_r.encode(A						, add_special_tokens=A	)

                           _UpperCAmelCase					:  str											=   tokenizer_r.convert_ids_to_tokens(A	)
                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_p.convert_ids_to_tokens(A	)

                           # it is expected that each Chinese character is not preceded by "##"
                           self.assertListEqual(A						, A	)
                           self.assertListEqual(A						, A	)

                           _UpperCAmelCase					:  Tuple											=   False
                           _UpperCAmelCase					:  List[str]											=   self.rust_tokenizer_class.from_pretrained(A						, **A	)
                           _UpperCAmelCase					:  Union[str, Any]											=   self.tokenizer_class.from_pretrained(A						, **A	)

                           _UpperCAmelCase					:  Optional[int]											=   tokenizer_r.encode(A						, add_special_tokens=A	)
                           _UpperCAmelCase					:  Union[str, Any]											=   tokenizer_p.encode(A						, add_special_tokens=A	)

                           _UpperCAmelCase					:  Tuple											=   tokenizer_r.convert_ids_to_tokens(A	)
                           _UpperCAmelCase					:  Optional[Any]											=   tokenizer_p.convert_ids_to_tokens(A	)

                           # it is expected that only the first Chinese character is not preceded by "##".
                           _UpperCAmelCase					:  Any											=   [
                               F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A	)
                           ]
                           self.assertListEqual(A						, A	)
                           self.assertListEqual(A						, A	)




'''simple docstring'''




import warnings

from .generation import FlaxGenerationMixin

class      lowerCamelCase_     (snake_case__   ):
      '''simple docstring'''

      warnings.warn(
          "Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will "
          "be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead."							,			snake_case__							,			)

