


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 					lowerCAmelCase__	(    A__					,   unittest.TestCase):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE        =			ElectraTokenizer
     __SCREAMING_SNAKE_CASE        =			ElectraTokenizerFast
     __SCREAMING_SNAKE_CASE        =			True
     __SCREAMING_SNAKE_CASE        =			True
     __SCREAMING_SNAKE_CASE        =			filter_non_english


     def 			_lowerCamelCase			(						self)      ->    Union[str, Any]:
            super().setUp()

            _A				:	int       =			[
                "[UNK]",
                "[CLS]",
                "[SEP]",
                "[PAD]",
                "[MASK]",
                "want",
                "##want",
                "##ed",
                "wa",
                "un",
                "runn",
                "##ing",
                ",",
                "low",
                "lowest",
            ]
            _A				:	Dict       =			os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["vocab_file"])
            with open(self.vocab_file			,						"w"			,						encoding="utf-8") as vocab_writer:
                   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase)      ->    str:
            _A				:	Optional[Any]       =			"UNwant\u00E9d,running"
            _A				:	Optional[int]       =			"unwanted, running"
            return input_text, output_text


     def 			_lowerCamelCase			(						self)      ->    List[Any]:
            _A				:	Optional[int]       =			self.tokenizer_class(self.vocab_file)

            _A				:	Union[str, Any]       =			tokenizer.tokenize("UNwant\u00E9d,running")
            self.assertListEqual(snake_case_			,						["un", "##want", "##ed", ",", "runn", "##ing"])
            self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_)			,						[9, 6, 7, 1_2, 1_0, 1_1])


     def 			_lowerCamelCase			(						self)      ->    Any:
            if not self.test_rust_tokenizer:
                   return

            _A				:	Tuple       =			self.get_tokenizer()
            _A				:	str       =			self.get_rust_tokenizer()

            _A				:	Any       =			"UNwant\u00E9d,running"

            _A				:	int       =			tokenizer.tokenize(snake_case_)
            _A				:	int       =			rust_tokenizer.tokenize(snake_case_)
            self.assertListEqual(snake_case_			,						snake_case_)

            _A				:	int       =			tokenizer.encode(snake_case_			,						add_special_tokens=snake_case_)
            _A				:	Any       =			rust_tokenizer.encode(snake_case_			,						add_special_tokens=snake_case_)
            self.assertListEqual(snake_case_			,						snake_case_)

            _A				:	int       =			self.get_rust_tokenizer()
            _A				:	str       =			tokenizer.encode(snake_case_)
            _A				:	Tuple       =			rust_tokenizer.encode(snake_case_)
            self.assertListEqual(snake_case_			,						snake_case_)

            # With lower casing
            _A				:	int       =			self.get_tokenizer(do_lower_case=snake_case_)
            _A				:	Tuple       =			self.get_rust_tokenizer(do_lower_case=snake_case_)

            _A				:	str       =			"UNwant\u00E9d,running"

            _A				:	List[str]       =			tokenizer.tokenize(snake_case_)
            _A				:	Dict       =			rust_tokenizer.tokenize(snake_case_)
            self.assertListEqual(snake_case_			,						snake_case_)

            _A				:	Optional[Any]       =			tokenizer.encode(snake_case_			,						add_special_tokens=snake_case_)
            _A				:	List[str]       =			rust_tokenizer.encode(snake_case_			,						add_special_tokens=snake_case_)
            self.assertListEqual(snake_case_			,						snake_case_)

            _A				:	List[str]       =			self.get_rust_tokenizer()
            _A				:	Any       =			tokenizer.encode(snake_case_)
            _A				:	Dict       =			rust_tokenizer.encode(snake_case_)
            self.assertListEqual(snake_case_			,						snake_case_)


     def 			_lowerCamelCase			(						self)      ->    Union[str, Any]:
            _A				:	Optional[int]       =			BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")			,						["ah", "\u535A", "\u63A8", "zz"])


     def 			_lowerCamelCase			(						self)      ->    str:
            _A				:	List[str]       =			BasicTokenizer(do_lower_case=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,						["hello", "!", "how", "are", "you", "?"])
            self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,						["hello"])


     def 			_lowerCamelCase			(						self)      ->    str:
            _A				:	int       =			BasicTokenizer(do_lower_case=snake_case_			,						strip_accents=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,						["hällo", "!", "how", "are", "you", "?"])
            self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,						["h\u00E9llo"])


     def 			_lowerCamelCase			(						self)      ->    int:
            _A				:	List[str]       =			BasicTokenizer(do_lower_case=snake_case_			,						strip_accents=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,						["hallo", "!", "how", "are", "you", "?"])
            self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,						["hello"])


     def 			_lowerCamelCase			(						self)      ->    Dict:
            _A				:	List[Any]       =			BasicTokenizer(do_lower_case=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,						["hallo", "!", "how", "are", "you", "?"])
            self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,						["hello"])


     def 			_lowerCamelCase			(						self)      ->    List[str]:
            _A				:	str       =			BasicTokenizer(do_lower_case=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,						["HeLLo", "!", "how", "Are", "yoU", "?"])


     def 			_lowerCamelCase			(						self)      ->    Any:
            _A				:	List[Any]       =			BasicTokenizer(do_lower_case=snake_case_			,						strip_accents=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,						["HäLLo", "!", "how", "Are", "yoU", "?"])


     def 			_lowerCamelCase			(						self)      ->    Tuple:
            _A				:	Optional[Any]       =			BasicTokenizer(do_lower_case=snake_case_			,						strip_accents=snake_case_)

            self.assertListEqual(
                tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,						["HaLLo", "!", "how", "Are", "yoU", "?"])


     def 			_lowerCamelCase			(						self)      ->    Dict:
            _A				:	Optional[int]       =			BasicTokenizer(do_lower_case=snake_case_			,						never_split=["[UNK]"])

            self.assertListEqual(
                tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")			,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])


     def 			_lowerCamelCase			(						self)      ->    List[str]:
            _A				:	List[Any]       =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

            _A				:	List[Any]       =			{}
            for i, token in enumerate(snake_case_):
                   _A				:	Optional[Any]       =			i
            _A				:	Tuple       =			WordpieceTokenizer(vocab=snake_case_			,						unk_token="[UNK]")

            self.assertListEqual(tokenizer.tokenize("")			,						[])

            self.assertListEqual(tokenizer.tokenize("unwanted running")			,						["un", "##want", "##ed", "runn", "##ing"])

            self.assertListEqual(tokenizer.tokenize("unwantedX running")			,						["[UNK]", "runn", "##ing"])


     def 			_lowerCamelCase			(						self)      ->    str:
            self.assertTrue(_is_whitespace(" "))
            self.assertTrue(_is_whitespace("\t"))
            self.assertTrue(_is_whitespace("\r"))
            self.assertTrue(_is_whitespace("\n"))
            self.assertTrue(_is_whitespace("\u00A0"))

            self.assertFalse(_is_whitespace("A"))
            self.assertFalse(_is_whitespace("-"))


     def 			_lowerCamelCase			(						self)      ->    Optional[int]:
            self.assertTrue(_is_control("\u0005"))

            self.assertFalse(_is_control("A"))
            self.assertFalse(_is_control(" "))
            self.assertFalse(_is_control("\t"))
            self.assertFalse(_is_control("\r"))


     def 			_lowerCamelCase			(						self)      ->    int:
            self.assertTrue(_is_punctuation("-"))
            self.assertTrue(_is_punctuation("$"))
            self.assertTrue(_is_punctuation("`"))
            self.assertTrue(_is_punctuation("."))

            self.assertFalse(_is_punctuation("A"))
            self.assertFalse(_is_punctuation(" "))


     def 			_lowerCamelCase			(						self)      ->    str:
            _A				:	int       =			self.get_tokenizer()
            _A				:	List[str]       =			self.get_rust_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(snake_case_) for t in ["Test", "\xad", "test"]]			,						[["[UNK]"], [], ["[UNK]"]])

            self.assertListEqual(
                [rust_tokenizer.tokenize(snake_case_) for t in ["Test", "\xad", "test"]]			,						[["[UNK]"], [], ["[UNK]"]])


     @slow
     def 			_lowerCamelCase			(						self)      ->    List[str]:
            _A				:	str       =			self.tokenizer_class.from_pretrained("google/electra-base-discriminator")

            _A				:	Optional[Any]       =			tokenizer.encode("sequence builders"			,						add_special_tokens=snake_case_)
            _A				:	List[Any]       =			tokenizer.encode("multi-sequence build"			,						add_special_tokens=snake_case_)

            _A				:	Union[str, Any]       =			tokenizer.build_inputs_with_special_tokens(snake_case_)
            _A				:	int       =			tokenizer.build_inputs_with_special_tokens(snake_case_			,						snake_case_)

            assert encoded_sentence == [1_0_1] + text + [1_0_2]
            assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


     def 			_lowerCamelCase			(						self)      ->    int:
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
                          _A				:	Optional[Any]       =			self.rust_tokenizer_class.from_pretrained(snake_case_			,						**snake_case_)

                          _A				:	Any       =			F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                          _A				:	Dict       =			tokenizer_r.encode_plus(
                              snake_case_			,						return_attention_mask=snake_case_			,						return_token_type_ids=snake_case_			,						return_offsets_mapping=snake_case_			,						add_special_tokens=snake_case_			,						)

                          _A				:	List[Any]       =			tokenizer_r.do_lower_case if hasattr(snake_case_			,						"do_lower_case") else False
                          _A				:	Dict       =			(
                              [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), "A"),
                                  ((1, 2), ","),
                                  ((3, 5), "na"),
                                  ((5, 6), "##ï"),
                                  ((6, 8), "##ve"),
                                  ((9, 1_5), tokenizer_r.mask_token),
                                  ((1_6, 2_1), "Allen"),
                                  ((2_1, 2_3), "##NL"),
                                  ((2_3, 2_4), "##P"),
                                  ((2_5, 3_3), "sentence"),
                                  ((3_3, 3_4), "."),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                              if not do_lower_case
                              else [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), "a"),
                                  ((1, 2), ","),
                                  ((3, 8), "naive"),
                                  ((9, 1_5), tokenizer_r.mask_token),
                                  ((1_6, 2_1), "allen"),
                                  ((2_1, 2_3), "##nl"),
                                  ((2_3, 2_4), "##p"),
                                  ((2_5, 3_3), "sentence"),
                                  ((3_3, 3_4), "."),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                          )

                          self.assertEqual(
                              [e[1] for e in expected_results]			,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
                          self.assertEqual([e[0] for e in expected_results]			,						tokens["offset_mapping"])


     def 			_lowerCamelCase			(						self)      ->    List[Any]:
            _A				:	List[str]       =			["的", "人", "有"]
            _A				:	List[str]       =			"".join(snake_case_)
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
                          _A				:	Any       =			True
                          _A				:	int       =			self.tokenizer_class.from_pretrained(snake_case_			,						**snake_case_)
                          _A				:	Any       =			self.rust_tokenizer_class.from_pretrained(snake_case_			,						**snake_case_)

                          _A				:	Dict       =			tokenizer_p.encode(snake_case_			,						add_special_tokens=snake_case_)
                          _A				:	List[str]       =			tokenizer_r.encode(snake_case_			,						add_special_tokens=snake_case_)

                          _A				:	Tuple       =			tokenizer_r.convert_ids_to_tokens(snake_case_)
                          _A				:	Any       =			tokenizer_p.convert_ids_to_tokens(snake_case_)

                          # it is expected that each Chinese character is not preceded by "##"
                          self.assertListEqual(snake_case_			,						snake_case_)
                          self.assertListEqual(snake_case_			,						snake_case_)

                          _A				:	Optional[int]       =			False
                          _A				:	Tuple       =			self.rust_tokenizer_class.from_pretrained(snake_case_			,						**snake_case_)
                          _A				:	List[str]       =			self.tokenizer_class.from_pretrained(snake_case_			,						**snake_case_)

                          _A				:	Dict       =			tokenizer_r.encode(snake_case_			,						add_special_tokens=snake_case_)
                          _A				:	Dict       =			tokenizer_p.encode(snake_case_			,						add_special_tokens=snake_case_)

                          _A				:	List[str]       =			tokenizer_r.convert_ids_to_tokens(snake_case_)
                          _A				:	Any       =			tokenizer_p.convert_ids_to_tokens(snake_case_)

                          # it is expected that only the first Chinese character is not preceded by "##".
                          _A				:	Any       =			[
                              F"##{token}" if idx != 0 else token for idx, token in enumerate(snake_case_)
                          ]
                          self.assertListEqual(snake_case_			,						snake_case_)
                          self.assertListEqual(snake_case_			,						snake_case_)




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase__						=			logging.get_logger(__name__)

lowerCAmelCase__						=			{'vocab_file': 'vocab.txt'}

lowerCAmelCase__						=			{
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

lowerCAmelCase__						=			{
    'tau/splinter-base': 5_12,
    'tau/splinter-base-qass': 5_12,
    'tau/splinter-large': 5_12,
    'tau/splinter-large-qass': 5_12,
}

lowerCAmelCase__						=			{
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}







class 					lowerCAmelCase__	(    a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE        =			VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_INIT_CONFIGURATION
     __SCREAMING_SNAKE_CASE        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE        =			SplinterTokenizer


     def __init__(						self			,						__lowerCamelCase=None			,						__lowerCamelCase=None			,						__lowerCamelCase=True			,						__lowerCamelCase="[UNK]"			,						__lowerCamelCase="[SEP]"			,						__lowerCamelCase="[PAD]"			,						__lowerCamelCase="[CLS]"			,						__lowerCamelCase="[MASK]"			,						__lowerCamelCase="[QUESTION]"			,						__lowerCamelCase=True			,						__lowerCamelCase=None			,						**__lowerCamelCase			,						)      ->    Union[str, Any]:
            super().__init__(
                __lowerCamelCase			,						tokenizer_file=__lowerCamelCase			,						do_lower_case=__lowerCamelCase			,						unk_token=__lowerCamelCase			,						sep_token=__lowerCamelCase			,						pad_token=__lowerCamelCase			,						cls_token=__lowerCamelCase			,						mask_token=__lowerCamelCase			,						tokenize_chinese_chars=__lowerCamelCase			,						strip_accents=__lowerCamelCase			,						additional_special_tokens=(question_token,)			,						**__lowerCamelCase			,						)

            _A				:	Tuple       =			json.loads(self.backend_tokenizer.normalizer.__getstate__())
            if (
                pre_tok_state.get("lowercase"			,						__lowerCamelCase) != do_lower_case
                or pre_tok_state.get("strip_accents"			,						__lowerCamelCase) != strip_accents
            ):
                   _A				:	Dict       =			getattr(__lowerCamelCase			,						pre_tok_state.pop("type"))
                   _A				:	List[str]       =			do_lower_case
                   _A				:	Any       =			strip_accents
                   _A				:	int       =			pre_tok_class(**__lowerCamelCase)

            _A				:	int       =			do_lower_case


     @property
     def 			_lowerCamelCase			(						self)      ->    str:
            return self.convert_tokens_to_ids(self.question_token)


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    List[int]:
            if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

            _A				:	Tuple       =			[self.cls_token_id]
            _A				:	str       =			[self.sep_token_id]
            _A				:	int       =			[self.question_token_id] + [self.convert_tokens_to_ids(".")]
            if self.padding_side == "right":
                   # Input is question-then-context
                   return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
            else:
                   # Input is context-then-question
                   return cls + token_ids_a + sep + token_ids_a + question_suffix + sep


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    List[int]:
            _A				:	str       =			[self.sep_token_id]
            _A				:	int       =			[self.cls_token_id]
            _A				:	Union[str, Any]       =			[self.question_token_id] + [self.convert_tokens_to_ids(".")]
            if token_ids_a is None:
                   return len(cls + token_ids_a + sep) * [0]

            if self.padding_side == "right":
                   # Input is question-then-context
                   return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
            else:
                   # Input is context-then-question
                   return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]


     def 			_lowerCamelCase			(						self			,						__lowerCamelCase			,						__lowerCamelCase = None)      ->    Tuple[str]:
            _A				:	Tuple       =			self._tokenizer.model.save(__lowerCamelCase			,						name=__lowerCamelCase)
            return tuple(__lowerCamelCase)
