import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 		a__						( A__   ,		unittest.TestCase ):
					A     	=							ElectraTokenizer
					A     	=							ElectraTokenizerFast
					A     	=							True
					A     	=							True
					A     	=							filter_non_english

					def     __UpperCamelCase       (   self				:	int   ):







							"""simple docstring"""






							super().setUp()

							SCREAMING_SNAKE_CASE_  :		Union[str, Any]    				=   [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							SCREAMING_SNAKE_CASE_  :		int    				=   os.path.join(self.tmpdirname,VOCAB_FILES_NAMES["vocab_file"]   )
							with open(self.vocab_file,"w",encoding="utf-8"   ) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )

					def     __UpperCamelCase       (   self				:	List[str],_A				:	Optional[int]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		str    				=   "UNwant\u00E9d,running"
							SCREAMING_SNAKE_CASE_  :		Optional[int]    				=   "unwanted, running"
							return input_text, output_text

					def     __UpperCamelCase       (   self				:	str   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int    				=   self.tokenizer_class(self.vocab_file   )

							SCREAMING_SNAKE_CASE_  :		List[Any]    				=   tokenizer.tokenize("UNwant\u00E9d,running"   )
							self.assertListEqual(_A,["un", "##want", "##ed", ",", "runn", "##ing"]   )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_A   ),[9, 6, 7, 12, 10, 11]   )

					def     __UpperCamelCase       (   self				:	int   ):







							"""simple docstring"""






							if not self.test_rust_tokenizer:
									return

							SCREAMING_SNAKE_CASE_  :		Any    				=   self.get_tokenizer()
							SCREAMING_SNAKE_CASE_  :		int    				=   self.get_rust_tokenizer()

							SCREAMING_SNAKE_CASE_  :		Dict    				=   "UNwant\u00E9d,running"

							SCREAMING_SNAKE_CASE_  :		Dict    				=   tokenizer.tokenize(_A   )
							SCREAMING_SNAKE_CASE_  :		List[str]    				=   rust_tokenizer.tokenize(_A   )
							self.assertListEqual(_A,_A   )

							SCREAMING_SNAKE_CASE_  :		List[str]    				=   tokenizer.encode(_A,add_special_tokens=_A   )
							SCREAMING_SNAKE_CASE_  :		str    				=   rust_tokenizer.encode(_A,add_special_tokens=_A   )
							self.assertListEqual(_A,_A   )

							SCREAMING_SNAKE_CASE_  :		Dict    				=   self.get_rust_tokenizer()
							SCREAMING_SNAKE_CASE_  :		Tuple    				=   tokenizer.encode(_A   )
							SCREAMING_SNAKE_CASE_  :		Tuple    				=   rust_tokenizer.encode(_A   )
							self.assertListEqual(_A,_A   )

							# With lower casing
							SCREAMING_SNAKE_CASE_  :		Tuple    				=   self.get_tokenizer(do_lower_case=_A   )
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]    				=   self.get_rust_tokenizer(do_lower_case=_A   )

							SCREAMING_SNAKE_CASE_  :		Tuple    				=   "UNwant\u00E9d,running"

							SCREAMING_SNAKE_CASE_  :		Any    				=   tokenizer.tokenize(_A   )
							SCREAMING_SNAKE_CASE_  :		Optional[int]    				=   rust_tokenizer.tokenize(_A   )
							self.assertListEqual(_A,_A   )

							SCREAMING_SNAKE_CASE_  :		Dict    				=   tokenizer.encode(_A,add_special_tokens=_A   )
							SCREAMING_SNAKE_CASE_  :		List[str]    				=   rust_tokenizer.encode(_A,add_special_tokens=_A   )
							self.assertListEqual(_A,_A   )

							SCREAMING_SNAKE_CASE_  :		List[Any]    				=   self.get_rust_tokenizer()
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]    				=   tokenizer.encode(_A   )
							SCREAMING_SNAKE_CASE_  :		Optional[int]    				=   rust_tokenizer.encode(_A   )
							self.assertListEqual(_A,_A   )

					def     __UpperCamelCase       (   self				:	str   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]    				=   BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"   ),["ah", "\u535A", "\u63A8", "zz"]   )

					def     __UpperCamelCase       (   self				:	Optional[Any]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]    				=   BasicTokenizer(do_lower_case=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   ),["hello", "!", "how", "are", "you", "?"]   )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ),["hello"]   )

					def     __UpperCamelCase       (   self				:	Optional[int]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[int]    				=   BasicTokenizer(do_lower_case=_A,strip_accents=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ),["hällo", "!", "how", "are", "you", "?"]   )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ),["h\u00E9llo"]   )

					def     __UpperCamelCase       (   self				:	int   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Any    				=   BasicTokenizer(do_lower_case=_A,strip_accents=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ),["hallo", "!", "how", "are", "you", "?"]   )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ),["hello"]   )

					def     __UpperCamelCase       (   self				:	Tuple   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int    				=   BasicTokenizer(do_lower_case=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ),["hallo", "!", "how", "are", "you", "?"]   )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   ),["hello"]   )

					def     __UpperCamelCase       (   self				:	Tuple   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Tuple    				=   BasicTokenizer(do_lower_case=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   ),["HeLLo", "!", "how", "Are", "yoU", "?"]   )

					def     __UpperCamelCase       (   self				:	List[str]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Tuple    				=   BasicTokenizer(do_lower_case=_A,strip_accents=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ),["HäLLo", "!", "how", "Are", "yoU", "?"]   )

					def     __UpperCamelCase       (   self				:	Optional[Any]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]    				=   BasicTokenizer(do_lower_case=_A,strip_accents=_A   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   ),["HaLLo", "!", "how", "Are", "yoU", "?"]   )

					def     __UpperCamelCase       (   self				:	Optional[int]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Any    				=   BasicTokenizer(do_lower_case=_A,never_split=["[UNK]"]   )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"   ),["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]   )

					def     __UpperCamelCase       (   self				:	Tuple   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Union[str, Any]    				=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							SCREAMING_SNAKE_CASE_  :		int    				=   {}
							for i, token in enumerate(_A   ):
									SCREAMING_SNAKE_CASE_  :		str    				=   i
							SCREAMING_SNAKE_CASE_  :		Tuple    				=   WordpieceTokenizer(vocab=_A,unk_token="[UNK]"   )

							self.assertListEqual(tokenizer.tokenize(""   ),[]   )

							self.assertListEqual(tokenizer.tokenize("unwanted running"   ),["un", "##want", "##ed", "runn", "##ing"]   )

							self.assertListEqual(tokenizer.tokenize("unwantedX running"   ),["[UNK]", "runn", "##ing"]   )

					def     __UpperCamelCase       (   self				:	Any   ):







							"""simple docstring"""






							self.assertTrue(_is_whitespace(" "   )   )
							self.assertTrue(_is_whitespace("\t"   )   )
							self.assertTrue(_is_whitespace("\r"   )   )
							self.assertTrue(_is_whitespace("\n"   )   )
							self.assertTrue(_is_whitespace("\u00A0"   )   )

							self.assertFalse(_is_whitespace("A"   )   )
							self.assertFalse(_is_whitespace("-"   )   )

					def     __UpperCamelCase       (   self				:	Optional[int]   ):







							"""simple docstring"""






							self.assertTrue(_is_control("\u0005"   )   )

							self.assertFalse(_is_control("A"   )   )
							self.assertFalse(_is_control(" "   )   )
							self.assertFalse(_is_control("\t"   )   )
							self.assertFalse(_is_control("\r"   )   )

					def     __UpperCamelCase       (   self				:	Dict   ):







							"""simple docstring"""






							self.assertTrue(_is_punctuation("-"   )   )
							self.assertTrue(_is_punctuation("$"   )   )
							self.assertTrue(_is_punctuation("`"   )   )
							self.assertTrue(_is_punctuation("."   )   )

							self.assertFalse(_is_punctuation("A"   )   )
							self.assertFalse(_is_punctuation(" "   )   )

					def     __UpperCamelCase       (   self				:	List[Any]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int    				=   self.get_tokenizer()
							SCREAMING_SNAKE_CASE_  :		Tuple    				=   self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_A   ) for t in ["Test", "\xad", "test"]],[["[UNK]"], [], ["[UNK]"]]   )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(_A   ) for t in ["Test", "\xad", "test"]],[["[UNK]"], [], ["[UNK]"]]   )

					@slow
					def     __UpperCamelCase       (   self				:	Union[str, Any]   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]    				=   self.tokenizer_class.from_pretrained("google/electra-base-discriminator"   )

							SCREAMING_SNAKE_CASE_  :		Dict    				=   tokenizer.encode("sequence builders",add_special_tokens=_A   )
							SCREAMING_SNAKE_CASE_  :		Any    				=   tokenizer.encode("multi-sequence build",add_special_tokens=_A   )

							SCREAMING_SNAKE_CASE_  :		Optional[Any]    				=   tokenizer.build_inputs_with_special_tokens(_A   )
							SCREAMING_SNAKE_CASE_  :		Any    				=   tokenizer.build_inputs_with_special_tokens(_A,_A   )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]

					def     __UpperCamelCase       (   self				:	Tuple   ):







							"""simple docstring"""






							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
											SCREAMING_SNAKE_CASE_  :		Tuple    				=   self.rust_tokenizer_class.from_pretrained(_A,**_A   )

											SCREAMING_SNAKE_CASE_  :		str    				=   F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
											SCREAMING_SNAKE_CASE_  :		Dict    				=   tokenizer_r.encode_plus(
											    _A,return_attention_mask=_A,return_token_type_ids=_A,return_offsets_mapping=_A,add_special_tokens=_A,)

											SCREAMING_SNAKE_CASE_  :		Dict    				=   tokenizer_r.do_lower_case if hasattr(_A,"do_lower_case"   ) else False
											SCREAMING_SNAKE_CASE_  :		List[Any]    				=   (
											    [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), "A"),
											        ((1, 2), ","),
											        ((3, 5), "na"),
											        ((5, 6), "##ï"),
											        ((6, 8), "##ve"),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), "Allen"),
											        ((21, 23), "##NL"),
											        ((23, 24), "##P"),
											        ((25, 33), "sentence"),
											        ((33, 34), "."),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											    if not do_lower_case
											    else [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), "a"),
											        ((1, 2), ","),
											        ((3, 8), "naive"),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), "allen"),
											        ((21, 23), "##nl"),
											        ((23, 24), "##p"),
											        ((25, 33), "sentence"),
											        ((33, 34), "."),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											)

											self.assertEqual(
											    [e[1] for e in expected_results],tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]   )   )
											self.assertEqual([e[0] for e in expected_results],tokens["offset_mapping"]   )






					def     __UpperCamelCase       (   self				:	str   ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		str    				=   ["的", "人", "有"]
							SCREAMING_SNAKE_CASE_  :		Dict    				=   "".join(_A   )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
											SCREAMING_SNAKE_CASE_  :		str    				=   True
											SCREAMING_SNAKE_CASE_  :		Dict    				=   self.tokenizer_class.from_pretrained(_A,**_A   )
											SCREAMING_SNAKE_CASE_  :		Optional[int]    				=   self.rust_tokenizer_class.from_pretrained(_A,**_A   )

											SCREAMING_SNAKE_CASE_  :		Tuple    				=   tokenizer_p.encode(_A,add_special_tokens=_A   )
											SCREAMING_SNAKE_CASE_  :		List[Any]    				=   tokenizer_r.encode(_A,add_special_tokens=_A   )

											SCREAMING_SNAKE_CASE_  :		List[Any]    				=   tokenizer_r.convert_ids_to_tokens(_A   )
											SCREAMING_SNAKE_CASE_  :		int    				=   tokenizer_p.convert_ids_to_tokens(_A   )

											# it is expected that each Chinese character is not preceded by "##"
											self.assertListEqual(_A,_A   )
											self.assertListEqual(_A,_A   )

											SCREAMING_SNAKE_CASE_  :		str    				=   False
											SCREAMING_SNAKE_CASE_  :		List[str]    				=   self.rust_tokenizer_class.from_pretrained(_A,**_A   )
											SCREAMING_SNAKE_CASE_  :		List[str]    				=   self.tokenizer_class.from_pretrained(_A,**_A   )

											SCREAMING_SNAKE_CASE_  :		Any    				=   tokenizer_r.encode(_A,add_special_tokens=_A   )
											SCREAMING_SNAKE_CASE_  :		Optional[Any]    				=   tokenizer_p.encode(_A,add_special_tokens=_A   )

											SCREAMING_SNAKE_CASE_  :		str    				=   tokenizer_r.convert_ids_to_tokens(_A   )
											SCREAMING_SNAKE_CASE_  :		Dict    				=   tokenizer_p.convert_ids_to_tokens(_A   )

											# it is expected that only the first Chinese character is not preceded by "##".
											SCREAMING_SNAKE_CASE_  :		int    				=   [
											    F'##{token}' if idx != 0 else token for idx, token in enumerate(_A   )
											]
											self.assertListEqual(_A,_A   )
											self.assertListEqual(_A,_A   )

import warnings

from ...utils import logging
from .image_processing_vit import ViTImageProcessor


__lowerCamelCase :     Tuple          =  logging.get_logger(__name__)






class 		a__						( A__ ):

					def __init__(   self				:	List[Any],*_A				:	Union[str, Any],**_A				:	List[Any]   ):







							"""simple docstring"""






							warnings.warn(
							    "The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
							    " use ViTImageProcessor instead.",_A,)
							super().__init__(*_A,**_A   )
