





'''simple docstring'''


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				lowerCAmelCase_						(						__magic_name__  ,unittest.TestCase						):
 __lowerCamelCase  :							Optional[Any]    					=   ElectraTokenizer
 __lowerCamelCase  :							Dict    					=   ElectraTokenizerFast
 __lowerCamelCase  :							List[Any]    					=   True
 __lowerCamelCase  :							Dict    					=   True
 __lowerCamelCase  :							List[str]    					=   filter_non_english




 def 							_snake_case				(		self						)      ->							Any:
     super().setUp()

     _lowerCAmelCase		     = [
         "[UNK]",
         "[CLS]",
         "[SEP]",
         "[PAD]",
         "[MASK]",
         "want",
         "##want",
         "##ed",
         "wa",
         "un",
         "runn",
         "##ing",
         ",",
         "low",
         "lowest",
     ]
     _lowerCAmelCase		     = os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES["vocab_file"]						)
     with open(self.vocab_file     ,       "w"     ,       encoding="utf-8"						) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)




 def 							_snake_case				(		self     ,       _lowerCAmelCase						)      ->							Tuple:
     _lowerCAmelCase		     = "UNwant\u00E9d,running"
     _lowerCAmelCase		     = "unwanted, running"
     return input_text, output_text




 def 							_snake_case				(		self						)      ->							int:
     _lowerCAmelCase		     = self.tokenizer_class(self.vocab_file						)

     _lowerCAmelCase		     = tokenizer.tokenize("UNwant\u00E9d,running"						)
     self.assertListEqual(_lowerCAmelCase     ,       ["un", "##want", "##ed", ",", "runn", "##ing"]						)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase						)     ,       [9, 6, 7, 12, 10, 11]						)




 def 							_snake_case				(		self						)      ->							int:
     if not self.test_rust_tokenizer:
         return

     _lowerCAmelCase		     = self.get_tokenizer()
     _lowerCAmelCase		     = self.get_rust_tokenizer()

     _lowerCAmelCase		     = "UNwant\u00E9d,running"

     _lowerCAmelCase		     = tokenizer.tokenize(_lowerCAmelCase						)
     _lowerCAmelCase		     = rust_tokenizer.tokenize(_lowerCAmelCase						)
     self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

     _lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
     _lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
     self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

     _lowerCAmelCase		     = self.get_rust_tokenizer()
     _lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase						)
     _lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase						)
     self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

     # With lower casing
     _lowerCAmelCase		     = self.get_tokenizer(do_lower_case=_lowerCAmelCase						)
     _lowerCAmelCase		     = self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase						)

     _lowerCAmelCase		     = "UNwant\u00E9d,running"

     _lowerCAmelCase		     = tokenizer.tokenize(_lowerCAmelCase						)
     _lowerCAmelCase		     = rust_tokenizer.tokenize(_lowerCAmelCase						)
     self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

     _lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
     _lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
     self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

     _lowerCAmelCase		     = self.get_rust_tokenizer()
     _lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase						)
     _lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase						)
     self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)




 def 							_snake_case				(		self						)      ->							int:
     _lowerCAmelCase		     = BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)     ,       ["ah", "\u535A", "\u63A8", "zz"]						)




 def 							_snake_case				(		self						)      ->							Optional[Any]:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)     ,       ["hello", "!", "how", "are", "you", "?"]						)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["hello"]						)




 def 							_snake_case				(		self						)      ->							Optional[Any]:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["hällo", "!", "how", "are", "you", "?"]						)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["h\u00E9llo"]						)




 def 							_snake_case				(		self						)      ->							Optional[Any]:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["hallo", "!", "how", "are", "you", "?"]						)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["hello"]						)




 def 							_snake_case				(		self						)      ->							Optional[Any]:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["hallo", "!", "how", "are", "you", "?"]						)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["hello"]						)




 def 							_snake_case				(		self						)      ->							int:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)     ,       ["HeLLo", "!", "how", "Are", "yoU", "?"]						)




 def 							_snake_case				(		self						)      ->							Union[str, Any]:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["HäLLo", "!", "how", "Are", "yoU", "?"]						)




 def 							_snake_case				(		self						)      ->							List[str]:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["HaLLo", "!", "how", "Are", "yoU", "?"]						)




 def 							_snake_case				(		self						)      ->							Any:
     _lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       never_split=["[UNK]"]						)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)     ,       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)




 def 							_snake_case				(		self						)      ->							Tuple:
     _lowerCAmelCase		     = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

     _lowerCAmelCase		     = {}
     for i, token in enumerate(_lowerCAmelCase						):
         _lowerCAmelCase		     = i
     _lowerCAmelCase		     = WordpieceTokenizer(vocab=_lowerCAmelCase     ,       unk_token="[UNK]"						)

     self.assertListEqual(tokenizer.tokenize(""						)     ,       []						)

     self.assertListEqual(tokenizer.tokenize("unwanted running"						)     ,       ["un", "##want", "##ed", "runn", "##ing"]						)

     self.assertListEqual(tokenizer.tokenize("unwantedX running"						)     ,       ["[UNK]", "runn", "##ing"]						)




 def 							_snake_case				(		self						)      ->							str:
     self.assertTrue(_is_whitespace(" "						)						)
     self.assertTrue(_is_whitespace("\t"						)						)
     self.assertTrue(_is_whitespace("\r"						)						)
     self.assertTrue(_is_whitespace("\n"						)						)
     self.assertTrue(_is_whitespace("\u00A0"						)						)

     self.assertFalse(_is_whitespace("A"						)						)
     self.assertFalse(_is_whitespace("-"						)						)




 def 							_snake_case				(		self						)      ->							str:
     self.assertTrue(_is_control("\u0005"						)						)

     self.assertFalse(_is_control("A"						)						)
     self.assertFalse(_is_control(" "						)						)
     self.assertFalse(_is_control("\t"						)						)
     self.assertFalse(_is_control("\r"						)						)




 def 							_snake_case				(		self						)      ->							List[str]:
     self.assertTrue(_is_punctuation("-"						)						)
     self.assertTrue(_is_punctuation("$"						)						)
     self.assertTrue(_is_punctuation("`"						)						)
     self.assertTrue(_is_punctuation("."						)						)

     self.assertFalse(_is_punctuation("A"						)						)
     self.assertFalse(_is_punctuation(" "						)						)




 def 							_snake_case				(		self						)      ->							List[Any]:
     _lowerCAmelCase		     = self.get_tokenizer()
     _lowerCAmelCase		     = self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase						) for t in ["Test", "\xad", "test"]]     ,       [["[UNK]"], [], ["[UNK]"]]						)

     self.assertListEqual(
         [rust_tokenizer.tokenize(_lowerCAmelCase						) for t in ["Test", "\xad", "test"]]     ,       [["[UNK]"], [], ["[UNK]"]]						)




 @slow
 def 							_snake_case				(		self						)      ->							Optional[int]:
     _lowerCAmelCase		     = self.tokenizer_class.from_pretrained("google/electra-base-discriminator"						)

     _lowerCAmelCase		     = tokenizer.encode("sequence builders"     ,       add_special_tokens=_lowerCAmelCase						)
     _lowerCAmelCase		     = tokenizer.encode("multi-sequence build"     ,       add_special_tokens=_lowerCAmelCase						)

     _lowerCAmelCase		     = tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase						)
     _lowerCAmelCase		     = tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase     ,       _lowerCAmelCase						)

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]




 def 							_snake_case				(		self						)      ->							int:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             _lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

             _lowerCAmelCase		     = f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
             _lowerCAmelCase		     = tokenizer_r.encode_plus(
                 _lowerCAmelCase     ,       return_attention_mask=_lowerCAmelCase     ,       return_token_type_ids=_lowerCAmelCase     ,       return_offsets_mapping=_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase     ,       )

             _lowerCAmelCase		     = tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase     ,       "do_lower_case"						) else False
             _lowerCAmelCase		     = (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "Allen"),
                     ((21, 23), "##NL"),
                     ((23, 24), "##P"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "allen"),
                     ((21, 23), "##nl"),
                     ((23, 24), "##p"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]     ,       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
             self.assertEqual([e[0] for e in expected_results]     ,       tokens["offset_mapping"]						)





 def 							_snake_case				(		self						)      ->							Tuple:
     _lowerCAmelCase		     = ["的", "人", "有"]
     _lowerCAmelCase		     = "".join(_lowerCAmelCase						)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             _lowerCAmelCase		     = True
             _lowerCAmelCase		     = self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)
             _lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

             _lowerCAmelCase		     = tokenizer_p.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
             _lowerCAmelCase		     = tokenizer_r.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)

             _lowerCAmelCase		     = tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase						)
             _lowerCAmelCase		     = tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase						)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)
             self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

             _lowerCAmelCase		     = False
             _lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)
             _lowerCAmelCase		     = self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

             _lowerCAmelCase		     = tokenizer_r.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
             _lowerCAmelCase		     = tokenizer_p.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)

             _lowerCAmelCase		     = tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase						)
             _lowerCAmelCase		     = tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase						)

             # it is expected that only the first Chinese character is not preceded by "##".
             _lowerCAmelCase		     = [
                 f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase						)
             ]
             self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)
             self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)









'''simple docstring'''


from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


_SCREAMING_SNAKE_CASE			=							logging.get_logger(__name__)

class 				lowerCAmelCase_						(						__magic_name__						):
 __lowerCamelCase  :							str    					=   "mt5"
 __lowerCamelCase  :							Dict    					=   ["past_key_values"]




 def __init__(		self     ,       _lowerCAmelCase=250112     ,       _lowerCAmelCase=512     ,       _lowerCAmelCase=64     ,       _lowerCAmelCase=1024     ,       _lowerCAmelCase=8     ,       _lowerCAmelCase=None     ,       _lowerCAmelCase=6     ,       _lowerCAmelCase=32     ,       _lowerCAmelCase=128     ,       _lowerCAmelCase=0.1     ,       _lowerCAmelCase=1E-6     ,       _lowerCAmelCase=1.0     ,       _lowerCAmelCase="gated-gelu"     ,       _lowerCAmelCase=True     ,       _lowerCAmelCase=True     ,       _lowerCAmelCase="T5Tokenizer"     ,       _lowerCAmelCase=False     ,       _lowerCAmelCase=0     ,       _lowerCAmelCase=1     ,       _lowerCAmelCase=0     ,       **_lowerCAmelCase     ,       )      ->							Optional[Any]:
     super().__init__(
         is_encoder_decoder=_lowerCAmelCase     ,       tokenizer_class=_lowerCAmelCase     ,       tie_word_embeddings=_lowerCAmelCase     ,       pad_token_id=_lowerCAmelCase     ,       eos_token_id=_lowerCAmelCase     ,       decoder_start_token_id=_lowerCAmelCase     ,       **_lowerCAmelCase     ,       )
     _lowerCAmelCase		     = vocab_size
     _lowerCAmelCase		     = d_model
     _lowerCAmelCase		     = d_kv
     _lowerCAmelCase		     = d_ff
     _lowerCAmelCase		     = num_layers
     _lowerCAmelCase		     = (
         num_decoder_layers if num_decoder_layers is not None else self.num_layers
     )  # default = symmetry
     _lowerCAmelCase		     = num_heads
     _lowerCAmelCase		     = relative_attention_num_buckets
     _lowerCAmelCase		     = relative_attention_max_distance
     _lowerCAmelCase		     = dropout_rate
     _lowerCAmelCase		     = layer_norm_epsilon
     _lowerCAmelCase		     = initializer_factor
     _lowerCAmelCase		     = feed_forward_proj
     _lowerCAmelCase		     = use_cache

     _lowerCAmelCase		     = self.feed_forward_proj.split("-"						)
     _lowerCAmelCase		     = act_info[-1]
     _lowerCAmelCase		     = act_info[0] == "gated"

     if len(_lowerCAmelCase						) > 1 and act_info[0] != "gated" or len(_lowerCAmelCase						) > 2:
         raise ValueError(
             f'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
             "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
             "'gated-gelu' or 'relu'"						)

     # for backwards compatibility
     if feed_forward_proj == "gated-gelu":
         _lowerCAmelCase		     = "gelu_new"




 @property
 def 							_snake_case				(		self						)      ->							Any:
     return self.d_model




 @property
 def 							_snake_case				(		self						)      ->							Optional[Any]:
     return self.num_heads





 @property
 def 							_snake_case				(		self						)      ->							List[str]:
     return self.num_layers







class 				lowerCAmelCase_						(						__magic_name__						):




 @property
 # Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.inputs
 def 							_snake_case				(		self						)      ->							Mapping[str, Mapping[int, str]]:
     _lowerCAmelCase		     = {
         "input_ids": {0: "batch", 1: "encoder_sequence"},
         "attention_mask": {0: "batch", 1: "encoder_sequence"},
     }
     if self.use_past:
         _lowerCAmelCase		     = "past_encoder_sequence + sequence"
         _lowerCAmelCase		     = {0: "batch"}
         _lowerCAmelCase		     = {0: "batch", 1: "past_decoder_sequence + sequence"}
     else:
         _lowerCAmelCase		     = {0: "batch", 1: "decoder_sequence"}
         _lowerCAmelCase		     = {0: "batch", 1: "decoder_sequence"}

     if self.use_past:
         self.fill_with_past_key_values_(_lowerCAmelCase     ,       direction="inputs"						)

     return common_inputs




 @property
 # Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.default_onnx_opset
 def 							_snake_case				(		self						)      ->							int:
     return 13





 @property
 def 							_snake_case				(		self						)      ->							float:
     return 5E-4


