





'''simple docstring'''


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				lowerCAmelCase_						(						__magic_name__  ,unittest.TestCase						):
	__lowerCamelCase  :							Optional[Any]    					=   ElectraTokenizer
	__lowerCamelCase  :							Dict    					=   ElectraTokenizerFast
	__lowerCamelCase  :							List[Any]    					=   True
	__lowerCamelCase  :							Dict    					=   True
	__lowerCamelCase  :							List[str]    					=   filter_non_english




	def 							_snake_case				(		self						)      ->							Any:
					super().setUp()

					_lowerCAmelCase		     = [
					    "[UNK]",
					    "[CLS]",
					    "[SEP]",
					    "[PAD]",
					    "[MASK]",
					    "want",
					    "##want",
					    "##ed",
					    "wa",
					    "un",
					    "runn",
					    "##ing",
					    ",",
					    "low",
					    "lowest",
					]
					_lowerCAmelCase		     = os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES["vocab_file"]						)
					with open(self.vocab_file     ,       "w"     ,       encoding="utf-8"						) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)




	def 							_snake_case				(		self     ,       _lowerCAmelCase						)      ->							Tuple:
					_lowerCAmelCase		     = "UNwant\u00E9d,running"
					_lowerCAmelCase		     = "unwanted, running"
					return input_text, output_text




	def 							_snake_case				(		self						)      ->							int:
					_lowerCAmelCase		     = self.tokenizer_class(self.vocab_file						)

					_lowerCAmelCase		     = tokenizer.tokenize("UNwant\u00E9d,running"						)
					self.assertListEqual(_lowerCAmelCase     ,       ["un", "##want", "##ed", ",", "runn", "##ing"]						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase						)     ,       [9, 6, 7, 12, 10, 11]						)




	def 							_snake_case				(		self						)      ->							int:
					if not self.test_rust_tokenizer:
									return

					_lowerCAmelCase		     = self.get_tokenizer()
					_lowerCAmelCase		     = self.get_rust_tokenizer()

					_lowerCAmelCase		     = "UNwant\u00E9d,running"

					_lowerCAmelCase		     = tokenizer.tokenize(_lowerCAmelCase						)
					_lowerCAmelCase		     = rust_tokenizer.tokenize(_lowerCAmelCase						)
					self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

					_lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
					_lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
					self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

					_lowerCAmelCase		     = self.get_rust_tokenizer()
					_lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase						)
					_lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase						)
					self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

					# With lower casing
					_lowerCAmelCase		     = self.get_tokenizer(do_lower_case=_lowerCAmelCase						)
					_lowerCAmelCase		     = self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase						)

					_lowerCAmelCase		     = "UNwant\u00E9d,running"

					_lowerCAmelCase		     = tokenizer.tokenize(_lowerCAmelCase						)
					_lowerCAmelCase		     = rust_tokenizer.tokenize(_lowerCAmelCase						)
					self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

					_lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
					_lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
					self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

					_lowerCAmelCase		     = self.get_rust_tokenizer()
					_lowerCAmelCase		     = tokenizer.encode(_lowerCAmelCase						)
					_lowerCAmelCase		     = rust_tokenizer.encode(_lowerCAmelCase						)
					self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)




	def 							_snake_case				(		self						)      ->							int:
					_lowerCAmelCase		     = BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)     ,       ["ah", "\u535A", "\u63A8", "zz"]						)




	def 							_snake_case				(		self						)      ->							Optional[Any]:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)     ,       ["hello", "!", "how", "are", "you", "?"]						)
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["hello"]						)




	def 							_snake_case				(		self						)      ->							Optional[Any]:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["hällo", "!", "how", "are", "you", "?"]						)
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["h\u00E9llo"]						)




	def 							_snake_case				(		self						)      ->							Optional[Any]:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["hallo", "!", "how", "are", "you", "?"]						)
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["hello"]						)




	def 							_snake_case				(		self						)      ->							Optional[Any]:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["hallo", "!", "how", "are", "you", "?"]						)
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)     ,       ["hello"]						)




	def 							_snake_case				(		self						)      ->							int:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)     ,       ["HeLLo", "!", "how", "Are", "yoU", "?"]						)




	def 							_snake_case				(		self						)      ->							Union[str, Any]:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["HäLLo", "!", "how", "Are", "yoU", "?"]						)




	def 							_snake_case				(		self						)      ->							List[str]:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       strip_accents=_lowerCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)     ,       ["HaLLo", "!", "how", "Are", "yoU", "?"]						)




	def 							_snake_case				(		self						)      ->							Any:
					_lowerCAmelCase		     = BasicTokenizer(do_lower_case=_lowerCAmelCase     ,       never_split=["[UNK]"]						)

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)     ,       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)




	def 							_snake_case				(		self						)      ->							Tuple:
					_lowerCAmelCase		     = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

					_lowerCAmelCase		     = {}
					for i, token in enumerate(_lowerCAmelCase						):
									_lowerCAmelCase		     = i
					_lowerCAmelCase		     = WordpieceTokenizer(vocab=_lowerCAmelCase     ,       unk_token="[UNK]"						)

					self.assertListEqual(tokenizer.tokenize(""						)     ,       []						)

					self.assertListEqual(tokenizer.tokenize("unwanted running"						)     ,       ["un", "##want", "##ed", "runn", "##ing"]						)

					self.assertListEqual(tokenizer.tokenize("unwantedX running"						)     ,       ["[UNK]", "runn", "##ing"]						)




	def 							_snake_case				(		self						)      ->							str:
					self.assertTrue(_is_whitespace(" "						)						)
					self.assertTrue(_is_whitespace("\t"						)						)
					self.assertTrue(_is_whitespace("\r"						)						)
					self.assertTrue(_is_whitespace("\n"						)						)
					self.assertTrue(_is_whitespace("\u00A0"						)						)

					self.assertFalse(_is_whitespace("A"						)						)
					self.assertFalse(_is_whitespace("-"						)						)




	def 							_snake_case				(		self						)      ->							str:
					self.assertTrue(_is_control("\u0005"						)						)

					self.assertFalse(_is_control("A"						)						)
					self.assertFalse(_is_control(" "						)						)
					self.assertFalse(_is_control("\t"						)						)
					self.assertFalse(_is_control("\r"						)						)




	def 							_snake_case				(		self						)      ->							List[str]:
					self.assertTrue(_is_punctuation("-"						)						)
					self.assertTrue(_is_punctuation("$"						)						)
					self.assertTrue(_is_punctuation("`"						)						)
					self.assertTrue(_is_punctuation("."						)						)

					self.assertFalse(_is_punctuation("A"						)						)
					self.assertFalse(_is_punctuation(" "						)						)




	def 							_snake_case				(		self						)      ->							List[Any]:
					_lowerCAmelCase		     = self.get_tokenizer()
					_lowerCAmelCase		     = self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase						) for t in ["Test", "\xad", "test"]]     ,       [["[UNK]"], [], ["[UNK]"]]						)

					self.assertListEqual(
					    [rust_tokenizer.tokenize(_lowerCAmelCase						) for t in ["Test", "\xad", "test"]]     ,       [["[UNK]"], [], ["[UNK]"]]						)




	@slow
	def 							_snake_case				(		self						)      ->							Optional[int]:
					_lowerCAmelCase		     = self.tokenizer_class.from_pretrained("google/electra-base-discriminator"						)

					_lowerCAmelCase		     = tokenizer.encode("sequence builders"     ,       add_special_tokens=_lowerCAmelCase						)
					_lowerCAmelCase		     = tokenizer.encode("multi-sequence build"     ,       add_special_tokens=_lowerCAmelCase						)

					_lowerCAmelCase		     = tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase						)
					_lowerCAmelCase		     = tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase     ,       _lowerCAmelCase						)

					assert encoded_sentence == [101] + text + [102]
					assert encoded_pair == [101] + text + [102] + text_a + [102]




	def 							_snake_case				(		self						)      ->							int:
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													_lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

													_lowerCAmelCase		     = f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													_lowerCAmelCase		     = tokenizer_r.encode_plus(
													    _lowerCAmelCase     ,       return_attention_mask=_lowerCAmelCase     ,       return_token_type_ids=_lowerCAmelCase     ,       return_offsets_mapping=_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase     ,       )

													_lowerCAmelCase		     = tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase     ,       "do_lower_case"						) else False
													_lowerCAmelCase		     = (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]     ,       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
													self.assertEqual([e[0] for e in expected_results]     ,       tokens["offset_mapping"]						)





	def 							_snake_case				(		self						)      ->							Tuple:
					_lowerCAmelCase		     = ["的", "人", "有"]
					_lowerCAmelCase		     = "".join(_lowerCAmelCase						)
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													_lowerCAmelCase		     = True
													_lowerCAmelCase		     = self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)
													_lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

													_lowerCAmelCase		     = tokenizer_p.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
													_lowerCAmelCase		     = tokenizer_r.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)

													_lowerCAmelCase		     = tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase						)
													_lowerCAmelCase		     = tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase						)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)
													self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)

													_lowerCAmelCase		     = False
													_lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)
													_lowerCAmelCase		     = self.tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

													_lowerCAmelCase		     = tokenizer_r.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)
													_lowerCAmelCase		     = tokenizer_p.encode(_lowerCAmelCase     ,       add_special_tokens=_lowerCAmelCase						)

													_lowerCAmelCase		     = tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase						)
													_lowerCAmelCase		     = tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase						)

													# it is expected that only the first Chinese character is not preceded by "##".
													_lowerCAmelCase		     = [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase						)
													]
													self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)
													self.assertListEqual(_lowerCAmelCase     ,       _lowerCAmelCase						)









'''simple docstring'''


from __future__ import annotations

from collections.abc import Iterable, Iterator
from typing import Any, Generic, TypeVar

_SCREAMING_SNAKE_CASE			=							TypeVar("T", bound=bool)

class 				lowerCAmelCase_						(						Generic[T]						):




 def __init__(		self     ,       _lowerCAmelCase						)      ->							None:
     _lowerCAmelCase		     = value
     _lowerCAmelCase		     = None
     _lowerCAmelCase		     = None




 @property
 def 							_snake_case				(		self						)      ->							T:
     return self._value





 @staticmethod
 def 							_snake_case				(		_lowerCAmelCase     ,       _lowerCAmelCase						)      ->							SkewNode[T] | None:
     if not roota:
         return roota

     if not roota:
         return roota

     if roota.value > roota.value:
         _lowerCAmelCase     ,     _lowerCAmelCase		     = roota, roota

     _lowerCAmelCase		     = roota
     _lowerCAmelCase		     = roota.right
     _lowerCAmelCase		     = roota.left
     _lowerCAmelCase		     = SkewNode.merge(_lowerCAmelCase     ,       _lowerCAmelCase						)

     return result







class 				lowerCAmelCase_						(						Generic[T]						):




 def __init__(		self     ,       _lowerCAmelCase = ()						)      ->							None:
     _lowerCAmelCase		     = None
     if data:
         for item in data:
             self.insert(_lowerCAmelCase						)




 def __bool__(		self						)      ->							bool:
     return self._root is not None




 def __iter__(		self						)      ->							Iterator[T]:
     _lowerCAmelCase		     = []
     while self:
         result.append(self.pop()						)

     # Pushing items back to the heap not to clear it.
     for item in result:
         self.insert(_lowerCAmelCase						)

     return iter(_lowerCAmelCase						)




 def 							_snake_case				(		self     ,       _lowerCAmelCase						)      ->							None:
     _lowerCAmelCase		     = SkewNode.merge(self._root     ,       SkewNode(_lowerCAmelCase						)						)




 def 							_snake_case				(		self						)      ->							T | None:
     _lowerCAmelCase		     = self.top()
     _lowerCAmelCase		     = (
         SkewNode.merge(self._root.left     ,       self._root.right						) if self._root else None
     )

     return result




 def 							_snake_case				(		self						)      ->							T:
     if not self._root:
         raise IndexError("Can't get top element for the empty heap."						)
     return self._root.value





 def 							_snake_case				(		self						)      ->							None:
     _lowerCAmelCase		     = None


if __name__ == "__main__":
       import doctest

       doctest.testmod()


