
import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class      _lowerCAmelCase (     __a ,				unittest.TestCase	):
				_lowercase      		=RealmTokenizer
				_lowercase      		=RealmTokenizerFast
				_lowercase      		=True
				_lowercase      		=True
				_lowercase      		=filter_non_english




				def 					__a						( self					)    ->				List[str]:
							super().setUp()

							lowerCAmelCase_      =     [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							lowerCAmelCase_      =     os.path.join(self.tmpdirname    ,    VOCAB_FILES_NAMES["vocab_file"]					)
							with open(self.vocab_file    ,    "w"    ,    encoding="utf-8"					) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]					)					)




				def 					__a						( self    ,    _UpperCamelCase					)    ->				List[str]:
							lowerCAmelCase_      =     "UNwant\u00E9d,running"
							lowerCAmelCase_      =     "unwanted, running"
							return input_text, output_text




				def 					__a						( self					)    ->				List[str]:
							lowerCAmelCase_      =     self.tokenizer_class(self.vocab_file					)

							lowerCAmelCase_      =     tokenizer.tokenize("UNwant\u00E9d,running"					)
							self.assertListEqual(_UpperCamelCase    ,    ["un", "##want", "##ed", ",", "runn", "##ing"]					)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase					)    ,    [9, 6, 7, 12, 10, 11]					)




				def 					__a						( self					)    ->				str:
							if not self.test_rust_tokenizer:
										return

							lowerCAmelCase_      =     self.get_tokenizer()
							lowerCAmelCase_      =     self.get_rust_tokenizer()

							lowerCAmelCase_      =     "UNwant\u00E9d,running"

							lowerCAmelCase_      =     tokenizer.tokenize(_UpperCamelCase					)
							lowerCAmelCase_      =     rust_tokenizer.tokenize(_UpperCamelCase					)
							self.assertListEqual(_UpperCamelCase    ,    _UpperCamelCase					)

							lowerCAmelCase_      =     tokenizer.encode(_UpperCamelCase    ,    add_special_tokens=_UpperCamelCase					)
							lowerCAmelCase_      =     rust_tokenizer.encode(_UpperCamelCase    ,    add_special_tokens=_UpperCamelCase					)
							self.assertListEqual(_UpperCamelCase    ,    _UpperCamelCase					)

							lowerCAmelCase_      =     self.get_rust_tokenizer()
							lowerCAmelCase_      =     tokenizer.encode(_UpperCamelCase					)
							lowerCAmelCase_      =     rust_tokenizer.encode(_UpperCamelCase					)
							self.assertListEqual(_UpperCamelCase    ,    _UpperCamelCase					)

							# With lower casing
							lowerCAmelCase_      =     self.get_tokenizer(do_lower_case=_UpperCamelCase					)
							lowerCAmelCase_      =     self.get_rust_tokenizer(do_lower_case=_UpperCamelCase					)

							lowerCAmelCase_      =     "UNwant\u00E9d,running"

							lowerCAmelCase_      =     tokenizer.tokenize(_UpperCamelCase					)
							lowerCAmelCase_      =     rust_tokenizer.tokenize(_UpperCamelCase					)
							self.assertListEqual(_UpperCamelCase    ,    _UpperCamelCase					)

							lowerCAmelCase_      =     tokenizer.encode(_UpperCamelCase    ,    add_special_tokens=_UpperCamelCase					)
							lowerCAmelCase_      =     rust_tokenizer.encode(_UpperCamelCase    ,    add_special_tokens=_UpperCamelCase					)
							self.assertListEqual(_UpperCamelCase    ,    _UpperCamelCase					)

							lowerCAmelCase_      =     self.get_rust_tokenizer()
							lowerCAmelCase_      =     tokenizer.encode(_UpperCamelCase					)
							lowerCAmelCase_      =     rust_tokenizer.encode(_UpperCamelCase					)
							self.assertListEqual(_UpperCamelCase    ,    _UpperCamelCase					)




				def 					__a						( self					)    ->				str:
							lowerCAmelCase_      =     BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"					)    ,    ["ah", "\u535A", "\u63A8", "zz"]					)




				def 					__a						( self					)    ->				List[Any]:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)    ,    ["hello", "!", "how", "are", "you", "?"]					)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)    ,    ["hello"]					)




				def 					__a						( self					)    ->				Tuple:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase    ,    strip_accents=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)    ,    ["hällo", "!", "how", "are", "you", "?"]					)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)    ,    ["h\u00E9llo"]					)




				def 					__a						( self					)    ->				Any:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase    ,    strip_accents=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)    ,    ["hallo", "!", "how", "are", "you", "?"]					)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)    ,    ["hello"]					)




				def 					__a						( self					)    ->				Union[str, Any]:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)    ,    ["hallo", "!", "how", "are", "you", "?"]					)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)    ,    ["hello"]					)




				def 					__a						( self					)    ->				int:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)    ,    ["HeLLo", "!", "how", "Are", "yoU", "?"]					)




				def 					__a						( self					)    ->				Union[str, Any]:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase    ,    strip_accents=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)    ,    ["HäLLo", "!", "how", "Are", "yoU", "?"]					)




				def 					__a						( self					)    ->				Optional[int]:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase    ,    strip_accents=_UpperCamelCase					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)    ,    ["HaLLo", "!", "how", "Are", "yoU", "?"]					)




				def 					__a						( self					)    ->				Optional[int]:
							lowerCAmelCase_      =     BasicTokenizer(do_lower_case=_UpperCamelCase    ,    never_split=["[UNK]"]					)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"					)    ,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]					)




				def 					__a						( self					)    ->				Union[str, Any]:
							lowerCAmelCase_      =     ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							lowerCAmelCase_      =     {}
							for i, token in enumerate(_UpperCamelCase					):
										lowerCAmelCase_      =     i
							lowerCAmelCase_      =     WordpieceTokenizer(vocab=_UpperCamelCase    ,    unk_token="[UNK]"					)

							self.assertListEqual(tokenizer.tokenize(""					)    ,    []					)

							self.assertListEqual(tokenizer.tokenize("unwanted running"					)    ,    ["un", "##want", "##ed", "runn", "##ing"]					)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"					)    ,    ["[UNK]", "runn", "##ing"]					)




				def 					__a						( self					)    ->				Any:
							self.assertTrue(_is_whitespace(" "					)					)
							self.assertTrue(_is_whitespace("\t"					)					)
							self.assertTrue(_is_whitespace("\r"					)					)
							self.assertTrue(_is_whitespace("\n"					)					)
							self.assertTrue(_is_whitespace("\u00A0"					)					)

							self.assertFalse(_is_whitespace("A"					)					)
							self.assertFalse(_is_whitespace("-"					)					)




				def 					__a						( self					)    ->				List[Any]:
							self.assertTrue(_is_control("\u0005"					)					)

							self.assertFalse(_is_control("A"					)					)
							self.assertFalse(_is_control(" "					)					)
							self.assertFalse(_is_control("\t"					)					)
							self.assertFalse(_is_control("\r"					)					)




				def 					__a						( self					)    ->				str:
							self.assertTrue(_is_punctuation("-"					)					)
							self.assertTrue(_is_punctuation("$"					)					)
							self.assertTrue(_is_punctuation("`"					)					)
							self.assertTrue(_is_punctuation("."					)					)

							self.assertFalse(_is_punctuation("A"					)					)
							self.assertFalse(_is_punctuation(" "					)					)




				def 					__a						( self					)    ->				List[Any]:
							lowerCAmelCase_      =     self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_UpperCamelCase					) for t in ["Test", "\xad", "test"]]    ,    [["[UNK]"], [], ["[UNK]"]]					)

							if self.test_rust_tokenizer:
										lowerCAmelCase_      =     self.get_rust_tokenizer()
										self.assertListEqual(
										    [rust_tokenizer.tokenize(_UpperCamelCase					) for t in ["Test", "\xad", "test"]]    ,    [["[UNK]"], [], ["[UNK]"]]					)




				@slow
				def 					__a						( self					)    ->				Any:
							lowerCAmelCase_      =     self.tokenizer_class.from_pretrained("bert-base-uncased"					)

							lowerCAmelCase_      =     tokenizer.encode("sequence builders"    ,    add_special_tokens=_UpperCamelCase					)
							lowerCAmelCase_      =     tokenizer.encode("multi-sequence build"    ,    add_special_tokens=_UpperCamelCase					)

							lowerCAmelCase_      =     tokenizer.build_inputs_with_special_tokens(_UpperCamelCase					)
							lowerCAmelCase_      =     tokenizer.build_inputs_with_special_tokens(_UpperCamelCase    ,    _UpperCamelCase					)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]




				def 					__a						( self					)    ->				Optional[int]:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
													lowerCAmelCase_      =     self.rust_tokenizer_class.from_pretrained(_UpperCamelCase    ,    **_UpperCamelCase					)

													lowerCAmelCase_      =     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
													lowerCAmelCase_      =     tokenizer_r.encode_plus(
													    _UpperCamelCase    ,    return_attention_mask=_UpperCamelCase    ,    return_token_type_ids=_UpperCamelCase    ,    return_offsets_mapping=_UpperCamelCase    ,    add_special_tokens=_UpperCamelCase    ,    )

													lowerCAmelCase_      =     tokenizer_r.do_lower_case if hasattr(_UpperCamelCase    ,    "do_lower_case"					) else False
													lowerCAmelCase_      =     (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]    ,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]					)					)
													self.assertEqual([e[0] for e in expected_results]    ,    tokens["offset_mapping"]					)




				@slow
				def 					__a						( self					)    ->				Dict:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
													lowerCAmelCase_      =     self.rust_tokenizer_class.from_pretrained(_UpperCamelCase    ,    **_UpperCamelCase					)
													lowerCAmelCase_      =     self.tokenizer_class.from_pretrained(_UpperCamelCase    ,    **_UpperCamelCase					)
													lowerCAmelCase_      =     [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

													lowerCAmelCase_      =     tokenizer_r.batch_encode_candidates(_UpperCamelCase    ,    max_length=10    ,    return_tensors="np"					)
													lowerCAmelCase_      =     tokenizer_p.batch_encode_candidates(_UpperCamelCase    ,    max_length=10    ,    return_tensors="np"					)

													lowerCAmelCase_      =     (2, 2, 10)

													self.assertEqual(encoded_sentence_r["input_ids"].shape    ,    _UpperCamelCase					)
													self.assertEqual(encoded_sentence_r["attention_mask"].shape    ,    _UpperCamelCase					)
													self.assertEqual(encoded_sentence_r["token_type_ids"].shape    ,    _UpperCamelCase					)

													self.assertEqual(encoded_sentence_p["input_ids"].shape    ,    _UpperCamelCase					)
													self.assertEqual(encoded_sentence_p["attention_mask"].shape    ,    _UpperCamelCase					)
													self.assertEqual(encoded_sentence_p["token_type_ids"].shape    ,    _UpperCamelCase					)





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


_A       =					{
    "configuration_xlm_prophetnet": ["XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLMProphetNetConfig"],
}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A       =					["XLMProphetNetTokenizer"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A       =					[
			    "XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "XLMProphetNetDecoder",
			    "XLMProphetNetEncoder",
			    "XLMProphetNetForCausalLM",
			    "XLMProphetNetForConditionalGeneration",
			    "XLMProphetNetModel",
			    "XLMProphetNetPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig

			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_xlm_prophetnet import (
						    XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
						    XLMProphetNetDecoder,
						    XLMProphetNetEncoder,
						    XLMProphetNetForCausalLM,
						    XLMProphetNetForConditionalGeneration,
						    XLMProphetNetModel,
						    XLMProphetNetPreTrainedModel,
						)

else:
			import sys

			_A       =					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



