
'''simple docstring'''


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 		__lowercase			(					_lowercase ,       unittest.TestCase      ):
   lowerCamelCase     :						Optional[int]		 =     RealmTokenizer
   lowerCamelCase     :						Optional[Any]		 =     RealmTokenizerFast
   lowerCamelCase     :						Optional[int]		 =     True
   lowerCamelCase     :						str		 =     True
   lowerCamelCase     :						List[str]		 =     filter_non_english

   def     UpperCAmelCase__	(self ):
        super().setUp()

        lowerCamelCase_   : Optional[int]       =						[
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "[PAD]",
            "[MASK]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        lowerCamelCase_   : List[str]       =						os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES['''vocab_file'''] )
        with open(self.vocab_file     ,				'''w'''     ,				encoding='''utf-8''' ) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )

   def     UpperCAmelCase__	(self     ,				A ):
        lowerCamelCase_   : List[Any]       =						"UNwant\u00E9d,running"
        lowerCamelCase_   : Dict       =						"unwanted, running"
        return input_text, output_text

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : str       =						self.tokenizer_class(self.vocab_file )

        lowerCamelCase_   : Any       =						tokenizer.tokenize('''UNwant\u00E9d,running''' )
        self.assertListEqual(A     ,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A )     ,				[9, 6, 7, 1_2, 1_0, 1_1] )

   def     UpperCAmelCase__	(self ):






        if not self.test_rust_tokenizer:
             return

        lowerCamelCase_   : Optional[Any]       =						self.get_tokenizer()
        lowerCamelCase_   : Any       =						self.get_rust_tokenizer()

        lowerCamelCase_   : Tuple       =						"UNwant\u00E9d,running"

        lowerCamelCase_   : List[str]       =						tokenizer.tokenize(A )
        lowerCamelCase_   : Union[str, Any]       =						rust_tokenizer.tokenize(A )
        self.assertListEqual(A     ,				A )

        lowerCamelCase_   : Dict       =						tokenizer.encode(A     ,				add_special_tokens=A )
        lowerCamelCase_   : Dict       =						rust_tokenizer.encode(A     ,				add_special_tokens=A )
        self.assertListEqual(A     ,				A )

        lowerCamelCase_   : List[Any]       =						self.get_rust_tokenizer()
        lowerCamelCase_   : Any       =						tokenizer.encode(A )
        lowerCamelCase_   : Optional[Any]       =						rust_tokenizer.encode(A )
        self.assertListEqual(A     ,				A )

        # With lower casing
        lowerCamelCase_   : Union[str, Any]       =						self.get_tokenizer(do_lower_case=A )
        lowerCamelCase_   : Optional[Any]       =						self.get_rust_tokenizer(do_lower_case=A )

        lowerCamelCase_   : int       =						"UNwant\u00E9d,running"

        lowerCamelCase_   : List[str]       =						tokenizer.tokenize(A )
        lowerCamelCase_   : Dict       =						rust_tokenizer.tokenize(A )
        self.assertListEqual(A     ,				A )

        lowerCamelCase_   : Optional[Any]       =						tokenizer.encode(A     ,				add_special_tokens=A )
        lowerCamelCase_   : Tuple       =						rust_tokenizer.encode(A     ,				add_special_tokens=A )
        self.assertListEqual(A     ,				A )

        lowerCamelCase_   : Dict       =						self.get_rust_tokenizer()
        lowerCamelCase_   : Tuple       =						tokenizer.encode(A )
        lowerCamelCase_   : int       =						rust_tokenizer.encode(A )
        self.assertListEqual(A     ,				A )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Any       =						BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )     ,				['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : str       =						BasicTokenizer(do_lower_case=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )     ,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,				['''hello'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Optional[int]       =						BasicTokenizer(do_lower_case=A     ,				strip_accents=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,				['''h\u00E9llo'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : List[Any]       =						BasicTokenizer(do_lower_case=A     ,				strip_accents=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,				['''hello'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Dict       =						BasicTokenizer(do_lower_case=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,				['''hello'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Any       =						BasicTokenizer(do_lower_case=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )     ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Union[str, Any]       =						BasicTokenizer(do_lower_case=A     ,				strip_accents=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : str       =						BasicTokenizer(do_lower_case=A     ,				strip_accents=A )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : List[Any]       =						BasicTokenizer(do_lower_case=A     ,				never_split=['''[UNK]'''] )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )     ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : int       =						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

        lowerCamelCase_   : Tuple       =						{}
        for i, token in enumerate(A ):
             lowerCamelCase_   : Union[str, Any]       =						i
        lowerCamelCase_   : List[str]       =						WordpieceTokenizer(vocab=A     ,				unk_token='''[UNK]''' )

        self.assertListEqual(tokenizer.tokenize('''''' )     ,				[] )

        self.assertListEqual(tokenizer.tokenize('''unwanted running''' )     ,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

        self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )     ,				['''[UNK]''', '''runn''', '''##ing'''] )

   def     UpperCAmelCase__	(self ):
        self.assertTrue(_is_whitespace(''' ''' ) )
        self.assertTrue(_is_whitespace('''\t''' ) )
        self.assertTrue(_is_whitespace('''\r''' ) )
        self.assertTrue(_is_whitespace('''\n''' ) )
        self.assertTrue(_is_whitespace('''\u00A0''' ) )

        self.assertFalse(_is_whitespace('''A''' ) )
        self.assertFalse(_is_whitespace('''-''' ) )

   def     UpperCAmelCase__	(self ):
        self.assertTrue(_is_control('''\u0005''' ) )

        self.assertFalse(_is_control('''A''' ) )
        self.assertFalse(_is_control(''' ''' ) )
        self.assertFalse(_is_control('''\t''' ) )
        self.assertFalse(_is_control('''\r''' ) )

   def     UpperCAmelCase__	(self ):
        self.assertTrue(_is_punctuation('''-''' ) )
        self.assertTrue(_is_punctuation('''$''' ) )
        self.assertTrue(_is_punctuation('''`''' ) )
        self.assertTrue(_is_punctuation('''.''' ) )

        self.assertFalse(_is_punctuation('''A''' ) )
        self.assertFalse(_is_punctuation(''' ''' ) )

   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Dict       =						self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(A ) for t in ['''Test''', '''\xad''', '''test''']]     ,				[['''[UNK]'''], [], ['''[UNK]''']] )

        if self.test_rust_tokenizer:
             lowerCamelCase_   : Dict       =						self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(A ) for t in ['''Test''', '''\xad''', '''test''']]     ,				[['''[UNK]'''], [], ['''[UNK]''']] )

   @slow
   def     UpperCAmelCase__	(self ):
        lowerCamelCase_   : Dict       =						self.tokenizer_class.from_pretrained('''bert-base-uncased''' )

        lowerCamelCase_   : List[str]       =						tokenizer.encode('''sequence builders'''     ,				add_special_tokens=A )
        lowerCamelCase_   : Optional[Any]       =						tokenizer.encode('''multi-sequence build'''     ,				add_special_tokens=A )

        lowerCamelCase_   : Optional[Any]       =						tokenizer.build_inputs_with_special_tokens(A )
        lowerCamelCase_   : List[str]       =						tokenizer.build_inputs_with_special_tokens(A     ,				A )

        assert encoded_sentence == [1_0_1] + text + [1_0_2]
        assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

   def     UpperCAmelCase__	(self ):






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
                  lowerCamelCase_   : Any       =						self.rust_tokenizer_class.from_pretrained(A     ,				**A )

                  lowerCamelCase_   : Dict       =						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                  lowerCamelCase_   : Union[str, Any]       =						tokenizer_r.encode_plus(
                      A     ,				return_attention_mask=A     ,				return_token_type_ids=A     ,				return_offsets_mapping=A     ,				add_special_tokens=A     ,				)

                  lowerCamelCase_   : Optional[Any]       =						tokenizer_r.do_lower_case if hasattr(A     ,				'''do_lower_case''' ) else False
                  lowerCamelCase_   : int       =						(
                      [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), "A"),
                          ((1, 2), ","),
                          ((3, 5), "na"),
                          ((5, 6), "##ï"),
                          ((6, 8), "##ve"),
                          ((9, 1_5), tokenizer_r.mask_token),
                          ((1_6, 2_1), "Allen"),
                          ((2_1, 2_3), "##NL"),
                          ((2_3, 2_4), "##P"),
                          ((2_5, 3_3), "sentence"),
                          ((3_3, 3_4), "."),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                      if not do_lower_case
                      else [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), "a"),
                          ((1, 2), ","),
                          ((3, 8), "naive"),
                          ((9, 1_5), tokenizer_r.mask_token),
                          ((1_6, 2_1), "allen"),
                          ((2_1, 2_3), "##nl"),
                          ((2_3, 2_4), "##p"),
                          ((2_5, 3_3), "sentence"),
                          ((3_3, 3_4), "."),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                  )

                  self.assertEqual(
                      [e[1] for e in expected_results]     ,				tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
                  self.assertEqual([e[0] for e in expected_results]     ,				tokens['''offset_mapping'''] )






   @slow
   def     UpperCAmelCase__	(self ):






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
                  lowerCamelCase_   : Optional[Any]       =						self.rust_tokenizer_class.from_pretrained(A     ,				**A )
                  lowerCamelCase_   : List[str]       =						self.tokenizer_class.from_pretrained(A     ,				**A )
                  lowerCamelCase_   : List[str]       =						[["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

                  lowerCamelCase_   : int       =						tokenizer_r.batch_encode_candidates(A     ,				max_length=1_0     ,				return_tensors='''np''' )
                  lowerCamelCase_   : List[str]       =						tokenizer_p.batch_encode_candidates(A     ,				max_length=1_0     ,				return_tensors='''np''' )

                  lowerCamelCase_   : Any       =						(2, 2, 1_0)

                  self.assertEqual(encoded_sentence_r['''input_ids'''].shape     ,				A )
                  self.assertEqual(encoded_sentence_r['''attention_mask'''].shape     ,				A )
                  self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape     ,				A )

                  self.assertEqual(encoded_sentence_p['''input_ids'''].shape     ,				A )
                  self.assertEqual(encoded_sentence_p['''attention_mask'''].shape     ,				A )
                  self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape     ,				A )




'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


__lowercase							:				List[str]					=       {
    '''configuration_sam''': [
        '''SAM_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''SamConfig''',
        '''SamMaskDecoderConfig''',
        '''SamPromptEncoderConfig''',
        '''SamVisionConfig''',
    ],
    '''processing_sam''': ['''SamProcessor'''],
}


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__lowercase							:				List[Any]					=       [
					    '''SAM_PRETRAINED_MODEL_ARCHIVE_LIST''',
					    '''SamModel''',
					    '''SamPreTrainedModel''',
					]
try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__lowercase							:				int					=       [
					    '''TF_SAM_PRETRAINED_MODEL_ARCHIVE_LIST''',
					    '''TFSamModel''',
					    '''TFSamPreTrainedModel''',
					]
try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__lowercase							:				str					=       ['''SamImageProcessor''']


if TYPE_CHECKING:
					from .configuration_sam import (
					    SAM_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    SamConfig,
					    SamMaskDecoderConfig,
					    SamPromptEncoderConfig,
					    SamVisionConfig,
					)
					from .processing_sam import SamProcessor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_sam import SAM_PRETRAINED_MODEL_ARCHIVE_LIST, SamModel, SamPreTrainedModel

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_sam import TF_SAM_PRETRAINED_MODEL_ARCHIVE_LIST, TFSamModel, TFSamPreTrainedModel

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .image_processing_sam import SamImageProcessor

else:
					import sys

					__lowercase							:				List[str]					=       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


