




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class    __lowerCAmelCase			(	UpperCAmelCase__ ,		unittest.TestCase			):
			snake_case_				:			List[str]								=					ElectraTokenizer
			snake_case_				:			Dict								=					ElectraTokenizerFast
			snake_case_				:			Any								=					True
			snake_case_				:			Union[str, Any]								=					True
			snake_case_				:			Optional[Any]								=					filter_non_english







			def UpperCamelCase      (	self   :      Union[str, Any]				):




									"""simple docstring"""


									super().setUp()

									_UpperCAmelCase													=					[
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									_UpperCAmelCase													=					os.path.join(self.tmpdirname   ,   VOCAB_FILES_NAMES["vocab_file"]				)
									with open(self.vocab_file   ,   "w"   ,   encoding="utf-8"				) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)







			def UpperCamelCase      (	self   :      int   ,   snake_case__   :      Dict				):




									"""simple docstring"""


									_UpperCAmelCase													=					"UNwant\u00E9d,running"
									_UpperCAmelCase													=					"unwanted, running"
									return input_text, output_text







			def UpperCamelCase      (	self   :      Tuple				):




									"""simple docstring"""


									_UpperCAmelCase													=					self.tokenizer_class(self.vocab_file				)

									_UpperCAmelCase													=					tokenizer.tokenize("UNwant\u00E9d,running"				)
									self.assertListEqual(snake_case__   ,   ["un", "##want", "##ed", ",", "runn", "##ing"]				)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__				)   ,   [9, 6, 7, 12, 10, 11]				)







			def UpperCamelCase      (	self   :      Any				):




									"""simple docstring"""


									if not self.test_rust_tokenizer:
															return

									_UpperCAmelCase													=					self.get_tokenizer()
									_UpperCAmelCase													=					self.get_rust_tokenizer()

									_UpperCAmelCase													=					"UNwant\u00E9d,running"

									_UpperCAmelCase													=					tokenizer.tokenize(snake_case__				)
									_UpperCAmelCase													=					rust_tokenizer.tokenize(snake_case__				)
									self.assertListEqual(snake_case__   ,   snake_case__				)

									_UpperCAmelCase													=					tokenizer.encode(snake_case__   ,   add_special_tokens=snake_case__				)
									_UpperCAmelCase													=					rust_tokenizer.encode(snake_case__   ,   add_special_tokens=snake_case__				)
									self.assertListEqual(snake_case__   ,   snake_case__				)

									_UpperCAmelCase													=					self.get_rust_tokenizer()
									_UpperCAmelCase													=					tokenizer.encode(snake_case__				)
									_UpperCAmelCase													=					rust_tokenizer.encode(snake_case__				)
									self.assertListEqual(snake_case__   ,   snake_case__				)

									# With lower casing
									_UpperCAmelCase													=					self.get_tokenizer(do_lower_case=snake_case__				)
									_UpperCAmelCase													=					self.get_rust_tokenizer(do_lower_case=snake_case__				)

									_UpperCAmelCase													=					"UNwant\u00E9d,running"

									_UpperCAmelCase													=					tokenizer.tokenize(snake_case__				)
									_UpperCAmelCase													=					rust_tokenizer.tokenize(snake_case__				)
									self.assertListEqual(snake_case__   ,   snake_case__				)

									_UpperCAmelCase													=					tokenizer.encode(snake_case__   ,   add_special_tokens=snake_case__				)
									_UpperCAmelCase													=					rust_tokenizer.encode(snake_case__   ,   add_special_tokens=snake_case__				)
									self.assertListEqual(snake_case__   ,   snake_case__				)

									_UpperCAmelCase													=					self.get_rust_tokenizer()
									_UpperCAmelCase													=					tokenizer.encode(snake_case__				)
									_UpperCAmelCase													=					rust_tokenizer.encode(snake_case__				)
									self.assertListEqual(snake_case__   ,   snake_case__				)







			def UpperCamelCase      (	self   :      List[str]				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)   ,   ["ah", "\u535A", "\u63A8", "zz"]				)







			def UpperCamelCase      (	self   :      List[str]				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)   ,   ["hello", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   ,   ["hello"]				)







			def UpperCamelCase      (	self   :      str				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__   ,   strip_accents=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   ,   ["hällo", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   ,   ["h\u00E9llo"]				)







			def UpperCamelCase      (	self   :      str				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__   ,   strip_accents=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   ,   ["hallo", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   ,   ["hello"]				)







			def UpperCamelCase      (	self   :      int				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   ,   ["hallo", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   ,   ["hello"]				)







			def UpperCamelCase      (	self   :      List[str]				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)   ,   ["HeLLo", "!", "how", "Are", "yoU", "?"]				)







			def UpperCamelCase      (	self   :      List[str]				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__   ,   strip_accents=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   ,   ["HäLLo", "!", "how", "Are", "yoU", "?"]				)







			def UpperCamelCase      (	self   :      Optional[int]				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__   ,   strip_accents=snake_case__				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   ,   ["HaLLo", "!", "how", "Are", "yoU", "?"]				)







			def UpperCamelCase      (	self   :      Tuple				):




									"""simple docstring"""


									_UpperCAmelCase													=					BasicTokenizer(do_lower_case=snake_case__   ,   never_split=["[UNK]"]				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)   ,   ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)







			def UpperCamelCase      (	self   :      Tuple				):




									"""simple docstring"""


									_UpperCAmelCase													=					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

									_UpperCAmelCase													=					{}
									for i, token in enumerate(snake_case__				):
															_UpperCAmelCase													=					i
									_UpperCAmelCase													=					WordpieceTokenizer(vocab=snake_case__   ,   unk_token="[UNK]"				)

									self.assertListEqual(tokenizer.tokenize(""				)   ,   []				)

									self.assertListEqual(tokenizer.tokenize("unwanted running"				)   ,   ["un", "##want", "##ed", "runn", "##ing"]				)

									self.assertListEqual(tokenizer.tokenize("unwantedX running"				)   ,   ["[UNK]", "runn", "##ing"]				)







			def UpperCamelCase      (	self   :      Dict				):




									"""simple docstring"""


									self.assertTrue(_is_whitespace(" "				)				)
									self.assertTrue(_is_whitespace("\t"				)				)
									self.assertTrue(_is_whitespace("\r"				)				)
									self.assertTrue(_is_whitespace("\n"				)				)
									self.assertTrue(_is_whitespace("\u00A0"				)				)

									self.assertFalse(_is_whitespace("A"				)				)
									self.assertFalse(_is_whitespace("-"				)				)







			def UpperCamelCase      (	self   :      Tuple				):




									"""simple docstring"""


									self.assertTrue(_is_control("\u0005"				)				)

									self.assertFalse(_is_control("A"				)				)
									self.assertFalse(_is_control(" "				)				)
									self.assertFalse(_is_control("\t"				)				)
									self.assertFalse(_is_control("\r"				)				)







			def UpperCamelCase      (	self   :      Union[str, Any]				):




									"""simple docstring"""


									self.assertTrue(_is_punctuation("-"				)				)
									self.assertTrue(_is_punctuation("$"				)				)
									self.assertTrue(_is_punctuation("`"				)				)
									self.assertTrue(_is_punctuation("."				)				)

									self.assertFalse(_is_punctuation("A"				)				)
									self.assertFalse(_is_punctuation(" "				)				)







			def UpperCamelCase      (	self   :      Tuple				):




									"""simple docstring"""


									_UpperCAmelCase													=					self.get_tokenizer()
									_UpperCAmelCase													=					self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case__				) for t in ["Test", "\xad", "test"]]   ,   [["[UNK]"], [], ["[UNK]"]]				)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(snake_case__				) for t in ["Test", "\xad", "test"]]   ,   [["[UNK]"], [], ["[UNK]"]]				)







			@slow
			def UpperCamelCase      (	self   :      str				):




									"""simple docstring"""


									_UpperCAmelCase													=					self.tokenizer_class.from_pretrained("google/electra-base-discriminator"				)

									_UpperCAmelCase													=					tokenizer.encode("sequence builders"   ,   add_special_tokens=snake_case__				)
									_UpperCAmelCase													=					tokenizer.encode("multi-sequence build"   ,   add_special_tokens=snake_case__				)

									_UpperCAmelCase													=					tokenizer.build_inputs_with_special_tokens(snake_case__				)
									_UpperCAmelCase													=					tokenizer.build_inputs_with_special_tokens(snake_case__   ,   snake_case__				)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]







			def UpperCamelCase      (	self   :      Optional[int]				):




									"""simple docstring"""


									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																					_UpperCAmelCase													=					self.rust_tokenizer_class.from_pretrained(snake_case__   ,   **snake_case__				)

																					_UpperCAmelCase													=					F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																					_UpperCAmelCase													=					tokenizer_r.encode_plus(
																					    snake_case__   ,   return_attention_mask=snake_case__   ,   return_token_type_ids=snake_case__   ,   return_offsets_mapping=snake_case__   ,   add_special_tokens=snake_case__   ,   )

																					_UpperCAmelCase													=					tokenizer_r.do_lower_case if hasattr(snake_case__   ,   "do_lower_case"				) else False
																					_UpperCAmelCase													=					(
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "A"),
																					        ((1, 2), ","),
																					        ((3, 5), "na"),
																					        ((5, 6), "##ï"),
																					        ((6, 8), "##ve"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "Allen"),
																					        ((21, 23), "##NL"),
																					        ((23, 24), "##P"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "a"),
																					        ((1, 2), ","),
																					        ((3, 8), "naive"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "allen"),
																					        ((21, 23), "##nl"),
																					        ((23, 24), "##p"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]   ,   tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
																					self.assertEqual([e[0] for e in expected_results]   ,   tokens["offset_mapping"]				)







			def UpperCamelCase      (	self   :      int				):




									"""simple docstring"""


									_UpperCAmelCase													=					["的", "人", "有"]
									_UpperCAmelCase													=					"".join(snake_case__				)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																					_UpperCAmelCase													=					True
																					_UpperCAmelCase													=					self.tokenizer_class.from_pretrained(snake_case__   ,   **snake_case__				)
																					_UpperCAmelCase													=					self.rust_tokenizer_class.from_pretrained(snake_case__   ,   **snake_case__				)

																					_UpperCAmelCase													=					tokenizer_p.encode(snake_case__   ,   add_special_tokens=snake_case__				)
																					_UpperCAmelCase													=					tokenizer_r.encode(snake_case__   ,   add_special_tokens=snake_case__				)

																					_UpperCAmelCase													=					tokenizer_r.convert_ids_to_tokens(snake_case__				)
																					_UpperCAmelCase													=					tokenizer_p.convert_ids_to_tokens(snake_case__				)

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(snake_case__   ,   snake_case__				)
																					self.assertListEqual(snake_case__   ,   snake_case__				)

																					_UpperCAmelCase													=					False
																					_UpperCAmelCase													=					self.rust_tokenizer_class.from_pretrained(snake_case__   ,   **snake_case__				)
																					_UpperCAmelCase													=					self.tokenizer_class.from_pretrained(snake_case__   ,   **snake_case__				)

																					_UpperCAmelCase													=					tokenizer_r.encode(snake_case__   ,   add_special_tokens=snake_case__				)
																					_UpperCAmelCase													=					tokenizer_p.encode(snake_case__   ,   add_special_tokens=snake_case__				)

																					_UpperCAmelCase													=					tokenizer_r.convert_ids_to_tokens(snake_case__				)
																					_UpperCAmelCase													=					tokenizer_p.convert_ids_to_tokens(snake_case__				)

																					# it is expected that only the first Chinese character is not preceded by "##".
																					_UpperCAmelCase													=					[
																					    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__				)
																					]
																					self.assertListEqual(snake_case__   ,   snake_case__				)
																					self.assertListEqual(snake_case__   ,   snake_case__				)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowercase_     :    int      =			{
    'configuration_videomae': ['VIDEOMAE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VideoMAEConfig'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_     :    List[str]      =			[
				    'VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'VideoMAEForPreTraining',
				    'VideoMAEModel',
				    'VideoMAEPreTrainedModel',
				    'VideoMAEForVideoClassification',
				]

try:
				if not is_vision_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_     :    Any      =			['VideoMAEFeatureExtractor']
				lowercase_     :    Dict      =			['VideoMAEImageProcessor']

if TYPE_CHECKING:
				from .configuration_videomae import VIDEOMAE_PRETRAINED_CONFIG_ARCHIVE_MAP, VideoMAEConfig

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_videomae import (
								    VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST,
								    VideoMAEForPreTraining,
								    VideoMAEForVideoClassification,
								    VideoMAEModel,
								    VideoMAEPreTrainedModel,
								)

				try:
								if not is_vision_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_videomae import VideoMAEFeatureExtractor
								from .image_processing_videomae import VideoMAEImageProcessor

else:
				import sys

				lowercase_     :    int      =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




