





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 			_lowerCamelCase	(      a	,				unittest.TestCase      ):



      """simple docstring"""






      UpperCAmelCase_  : int				      =RealmTokenizer
      UpperCAmelCase_  : Optional[Any]				      =RealmTokenizerFast
      UpperCAmelCase_  : Tuple				      =True
      UpperCAmelCase_  : str				      =True
      UpperCAmelCase_  : str				      =filter_non_english







      def   UpperCAmelCase						(	self				)    ->				List[Any]:






        '''simple docstring'''






        super().setUp()

        __snake_case				:			Tuple            =    [
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "[PAD]",
            "[MASK]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        __snake_case				:			List[str]            =    os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["vocab_file"]				)
        with open(self.vocab_file			,  "w"			,  encoding="utf-8"				) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)







      def   UpperCAmelCase						(	self			,  UpperCAmelCase				)    ->				Union[str, Any]:






        '''simple docstring'''






        __snake_case				:			Optional[Any]            =    "UNwant\u00E9d,running"
        __snake_case				:			Any            =    "unwanted, running"
        return input_text, output_text







      def   UpperCAmelCase						(	self				)    ->				Any:






        '''simple docstring'''






        __snake_case				:			Any            =    self.tokenizer_class(self.vocab_file				)

        __snake_case				:			Optional[Any]            =    tokenizer.tokenize("UNwant\u00E9d,running"				)
        self.assertListEqual(UpperCAmelCase			,  ["un", "##want", "##ed", ",", "runn", "##ing"]				)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase				)			,  [9, 6, 7, 12, 10, 11]				)







      def   UpperCAmelCase						(	self				)    ->				List[Any]:






        '''simple docstring'''






        if not self.test_rust_tokenizer:
          return

        __snake_case				:			Union[str, Any]            =    self.get_tokenizer()
        __snake_case				:			int            =    self.get_rust_tokenizer()

        __snake_case				:			Tuple            =    "UNwant\u00E9d,running"

        __snake_case				:			int            =    tokenizer.tokenize(UpperCAmelCase				)
        __snake_case				:			List[str]            =    rust_tokenizer.tokenize(UpperCAmelCase				)
        self.assertListEqual(UpperCAmelCase			,  UpperCAmelCase				)

        __snake_case				:			str            =    tokenizer.encode(UpperCAmelCase			,  add_special_tokens=UpperCAmelCase				)
        __snake_case				:			Optional[Any]            =    rust_tokenizer.encode(UpperCAmelCase			,  add_special_tokens=UpperCAmelCase				)
        self.assertListEqual(UpperCAmelCase			,  UpperCAmelCase				)

        __snake_case				:			Optional[int]            =    self.get_rust_tokenizer()
        __snake_case				:			Optional[Any]            =    tokenizer.encode(UpperCAmelCase				)
        __snake_case				:			Tuple            =    rust_tokenizer.encode(UpperCAmelCase				)
        self.assertListEqual(UpperCAmelCase			,  UpperCAmelCase				)

        # With lower casing
        __snake_case				:			List[str]            =    self.get_tokenizer(do_lower_case=UpperCAmelCase				)
        __snake_case				:			Optional[int]            =    self.get_rust_tokenizer(do_lower_case=UpperCAmelCase				)

        __snake_case				:			str            =    "UNwant\u00E9d,running"

        __snake_case				:			Tuple            =    tokenizer.tokenize(UpperCAmelCase				)
        __snake_case				:			str            =    rust_tokenizer.tokenize(UpperCAmelCase				)
        self.assertListEqual(UpperCAmelCase			,  UpperCAmelCase				)

        __snake_case				:			Optional[int]            =    tokenizer.encode(UpperCAmelCase			,  add_special_tokens=UpperCAmelCase				)
        __snake_case				:			Optional[int]            =    rust_tokenizer.encode(UpperCAmelCase			,  add_special_tokens=UpperCAmelCase				)
        self.assertListEqual(UpperCAmelCase			,  UpperCAmelCase				)

        __snake_case				:			Optional[Any]            =    self.get_rust_tokenizer()
        __snake_case				:			Tuple            =    tokenizer.encode(UpperCAmelCase				)
        __snake_case				:			Tuple            =    rust_tokenizer.encode(UpperCAmelCase				)
        self.assertListEqual(UpperCAmelCase			,  UpperCAmelCase				)







      def   UpperCAmelCase						(	self				)    ->				Dict:






        '''simple docstring'''






        __snake_case				:			Dict            =    BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)			,  ["ah", "\u535A", "\u63A8", "zz"]				)







      def   UpperCAmelCase						(	self				)    ->				str:






        '''simple docstring'''






        __snake_case				:			Union[str, Any]            =    BasicTokenizer(do_lower_case=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)			,  ["hello", "!", "how", "are", "you", "?"]				)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)			,  ["hello"]				)







      def   UpperCAmelCase						(	self				)    ->				int:






        '''simple docstring'''






        __snake_case				:			Dict            =    BasicTokenizer(do_lower_case=UpperCAmelCase			,  strip_accents=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)			,  ["hällo", "!", "how", "are", "you", "?"]				)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)			,  ["h\u00E9llo"]				)







      def   UpperCAmelCase						(	self				)    ->				Tuple:






        '''simple docstring'''






        __snake_case				:			Union[str, Any]            =    BasicTokenizer(do_lower_case=UpperCAmelCase			,  strip_accents=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)			,  ["hallo", "!", "how", "are", "you", "?"]				)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)			,  ["hello"]				)







      def   UpperCAmelCase						(	self				)    ->				List[str]:






        '''simple docstring'''






        __snake_case				:			Any            =    BasicTokenizer(do_lower_case=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)			,  ["hallo", "!", "how", "are", "you", "?"]				)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)			,  ["hello"]				)







      def   UpperCAmelCase						(	self				)    ->				Any:






        '''simple docstring'''






        __snake_case				:			str            =    BasicTokenizer(do_lower_case=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)			,  ["HeLLo", "!", "how", "Are", "yoU", "?"]				)







      def   UpperCAmelCase						(	self				)    ->				Optional[int]:






        '''simple docstring'''






        __snake_case				:			Tuple            =    BasicTokenizer(do_lower_case=UpperCAmelCase			,  strip_accents=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)			,  ["HäLLo", "!", "how", "Are", "yoU", "?"]				)







      def   UpperCAmelCase						(	self				)    ->				List[Any]:






        '''simple docstring'''






        __snake_case				:			Tuple            =    BasicTokenizer(do_lower_case=UpperCAmelCase			,  strip_accents=UpperCAmelCase				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)			,  ["HaLLo", "!", "how", "Are", "yoU", "?"]				)







      def   UpperCAmelCase						(	self				)    ->				str:






        '''simple docstring'''






        __snake_case				:			List[str]            =    BasicTokenizer(do_lower_case=UpperCAmelCase			,  never_split=["[UNK]"]				)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)			,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)







      def   UpperCAmelCase						(	self				)    ->				Dict:






        '''simple docstring'''






        __snake_case				:			List[str]            =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

        __snake_case				:			Optional[int]            =    {}
        for i, token in enumerate(UpperCAmelCase				):
          __snake_case				:			Optional[Any]            =    i
        __snake_case				:			List[str]            =    WordpieceTokenizer(vocab=UpperCAmelCase			,  unk_token="[UNK]"				)

        self.assertListEqual(tokenizer.tokenize(""				)			,  []				)

        self.assertListEqual(tokenizer.tokenize("unwanted running"				)			,  ["un", "##want", "##ed", "runn", "##ing"]				)

        self.assertListEqual(tokenizer.tokenize("unwantedX running"				)			,  ["[UNK]", "runn", "##ing"]				)







      def   UpperCAmelCase						(	self				)    ->				Union[str, Any]:






        '''simple docstring'''






        self.assertTrue(_is_whitespace(" "				)				)
        self.assertTrue(_is_whitespace("\t"				)				)
        self.assertTrue(_is_whitespace("\r"				)				)
        self.assertTrue(_is_whitespace("\n"				)				)
        self.assertTrue(_is_whitespace("\u00A0"				)				)

        self.assertFalse(_is_whitespace("A"				)				)
        self.assertFalse(_is_whitespace("-"				)				)







      def   UpperCAmelCase						(	self				)    ->				int:






        '''simple docstring'''






        self.assertTrue(_is_control("\u0005"				)				)

        self.assertFalse(_is_control("A"				)				)
        self.assertFalse(_is_control(" "				)				)
        self.assertFalse(_is_control("\t"				)				)
        self.assertFalse(_is_control("\r"				)				)







      def   UpperCAmelCase						(	self				)    ->				List[Any]:






        '''simple docstring'''






        self.assertTrue(_is_punctuation("-"				)				)
        self.assertTrue(_is_punctuation("$"				)				)
        self.assertTrue(_is_punctuation("`"				)				)
        self.assertTrue(_is_punctuation("."				)				)

        self.assertFalse(_is_punctuation("A"				)				)
        self.assertFalse(_is_punctuation(" "				)				)







      def   UpperCAmelCase						(	self				)    ->				Optional[Any]:






        '''simple docstring'''






        __snake_case				:			Union[str, Any]            =    self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(UpperCAmelCase				) for t in ["Test", "\xad", "test"]]			,  [["[UNK]"], [], ["[UNK]"]]				)

        if self.test_rust_tokenizer:
          __snake_case				:			Optional[int]            =    self.get_rust_tokenizer()
          self.assertListEqual(
              [rust_tokenizer.tokenize(UpperCAmelCase				) for t in ["Test", "\xad", "test"]]			,  [["[UNK]"], [], ["[UNK]"]]				)







      @slow
      def   UpperCAmelCase						(	self				)    ->				Union[str, Any]:






        '''simple docstring'''






        __snake_case				:			Any            =    self.tokenizer_class.from_pretrained("bert-base-uncased"				)

        __snake_case				:			Optional[int]            =    tokenizer.encode("sequence builders"			,  add_special_tokens=UpperCAmelCase				)
        __snake_case				:			List[str]            =    tokenizer.encode("multi-sequence build"			,  add_special_tokens=UpperCAmelCase				)

        __snake_case				:			int            =    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase				)
        __snake_case				:			List[str]            =    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			,  UpperCAmelCase				)

        assert encoded_sentence == [101] + text + [102]
        assert encoded_pair == [101] + text + [102] + text_a + [102]







      def   UpperCAmelCase						(	self				)    ->				Dict:






        '''simple docstring'''






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
            __snake_case				:			Dict            =    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase			,  **UpperCAmelCase				)

            __snake_case				:			Dict            =    F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
            __snake_case				:			List[str]            =    tokenizer_r.encode_plus(
                UpperCAmelCase			,  return_attention_mask=UpperCAmelCase			,  return_token_type_ids=UpperCAmelCase			,  return_offsets_mapping=UpperCAmelCase			,  add_special_tokens=UpperCAmelCase			,  )

            __snake_case				:			Optional[Any]            =    tokenizer_r.do_lower_case if hasattr(UpperCAmelCase			,  "do_lower_case"				) else False
            __snake_case				:			Any            =    (
                [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "A"),
                    ((1, 2), ","),
                    ((3, 5), "na"),
                    ((5, 6), "##ï"),
                    ((6, 8), "##ve"),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), "Allen"),
                    ((21, 23), "##NL"),
                    ((23, 24), "##P"),
                    ((25, 33), "sentence"),
                    ((33, 34), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
                if not do_lower_case
                else [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "a"),
                    ((1, 2), ","),
                    ((3, 8), "naive"),
                    ((9, 15), tokenizer_r.mask_token),
                    ((16, 21), "allen"),
                    ((21, 23), "##nl"),
                    ((23, 24), "##p"),
                    ((25, 33), "sentence"),
                    ((33, 34), "."),
                    ((0, 0), tokenizer_r.sep_token),
                ]
            )

            self.assertEqual(
                [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
            self.assertEqual([e[0] for e in expected_results]			,  tokens["offset_mapping"]				)







      @slow
      def   UpperCAmelCase						(	self				)    ->				List[Any]:






        '''simple docstring'''






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
            __snake_case				:			Any            =    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase			,  **UpperCAmelCase				)
            __snake_case				:			List[str]            =    self.tokenizer_class.from_pretrained(UpperCAmelCase			,  **UpperCAmelCase				)
            __snake_case				:			List[str]            =    [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

            __snake_case				:			int            =    tokenizer_r.batch_encode_candidates(UpperCAmelCase			,  max_length=10			,  return_tensors="np"				)
            __snake_case				:			Optional[int]            =    tokenizer_p.batch_encode_candidates(UpperCAmelCase			,  max_length=10			,  return_tensors="np"				)

            __snake_case				:			Tuple            =    (2, 2, 10)

            self.assertEqual(encoded_sentence_r["input_ids"].shape			,  UpperCAmelCase				)
            self.assertEqual(encoded_sentence_r["attention_mask"].shape			,  UpperCAmelCase				)
            self.assertEqual(encoded_sentence_r["token_type_ids"].shape			,  UpperCAmelCase				)

            self.assertEqual(encoded_sentence_p["input_ids"].shape			,  UpperCAmelCase				)
            self.assertEqual(encoded_sentence_p["attention_mask"].shape			,  UpperCAmelCase				)
            self.assertEqual(encoded_sentence_p["token_type_ids"].shape			,  UpperCAmelCase				)








import warnings

from ...utils import logging
from .image_processing_maskformer import MaskFormerImageProcessor


_UpperCamelCase    =	logging.get_logger(__name__)




class 			_lowerCamelCase	(      a      ):



      """simple docstring"""






      def __init__(	self			,  *UpperCAmelCase			,  **UpperCAmelCase				)    ->				None:






        '''simple docstring'''






        warnings.warn(
            "The class MaskFormerFeatureExtractor is deprecated and will be removed in version 5 of Transformers."
            " Please use MaskFormerImageProcessor instead."			,  UpperCAmelCase			,  )
        super().__init__(*UpperCAmelCase			,  **UpperCAmelCase				)

