




'''simple docstring'''







import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
   import torch

if is_vision_available():
   import PIL

   from transformers import FlavaImageProcessor
   from transformers.image_utils import PILImageResampling
   from transformers.models.flava.image_processing_flava import (
       FLAVA_CODEBOOK_MEAN,
       FLAVA_CODEBOOK_STD,
       FLAVA_IMAGE_MEAN,
       FLAVA_IMAGE_STD,
   )
else:
   A			   =A			   =A			   =A			   =None


class       _a  (     unittest.TestCase			):
       def __init__(     self						:     Any       ,    lowercase						:     Optional[Any]       ,    lowercase						:     Optional[Any]=7       ,    lowercase						:     Union[str, Any]=3       ,    lowercase						:     List[str]=30       ,    lowercase						:     str=400       ,    lowercase						:     str=True       ,    lowercase						:     Optional[Any]=None       ,    lowercase						:     Optional[int]=True       ,    lowercase						:     Any=None       ,    lowercase						:     Any=None       ,    lowercase						:     Dict=True       ,    lowercase						:     List[str]=1 / 255       ,    lowercase						:     Optional[int]=True       ,    lowercase						:     Optional[int]=FLAVA_IMAGE_MEAN       ,    lowercase						:     Tuple=FLAVA_IMAGE_STD       ,    lowercase						:     Any=14       ,    lowercase						:     List[str]=75       ,    lowercase						:     int=None       ,    lowercase						:     List[Any]=16       ,    lowercase						:     List[Any]=0.3       ,    lowercase						:     str=None       ,    lowercase						:     Any=True       ,    lowercase						:     Optional[int]=None       ,    lowercase						:     List[str]=None       ,    lowercase						:     Dict=True       ,    lowercase						:     Any=None       ,    lowercase						:     List[str]=True       ,    lowercase						:     Tuple=True       ,    lowercase						:     Optional[Any]=FLAVA_CODEBOOK_MEAN       ,    lowercase						:     Any=FLAVA_CODEBOOK_STD       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		size if size is not None else {'height': 224, 'width': 224}
           UpperCAmelCase      	=		crop_size if crop_size is not None else {'height': 224, 'width': 224}
           UpperCAmelCase      	=		codebook_size if codebook_size is not None else {'height': 112, 'width': 112}
           UpperCAmelCase      	=		codebook_crop_size if codebook_crop_size is not None else {'height': 112, 'width': 112}

           UpperCAmelCase      	=		parent
           UpperCAmelCase      	=		batch_size
           UpperCAmelCase      	=		num_channels
           UpperCAmelCase      	=		do_resize
           UpperCAmelCase      	=		do_rescale
           UpperCAmelCase      	=		rescale_factor
           UpperCAmelCase      	=		min_resolution
           UpperCAmelCase      	=		max_resolution
           UpperCAmelCase      	=		size
           UpperCAmelCase      	=		resample if resample is not None else PILImageResampling.BICUBIC
           UpperCAmelCase      	=		do_normalize
           UpperCAmelCase      	=		image_mean
           UpperCAmelCase      	=		image_std
           UpperCAmelCase      	=		do_center_crop
           UpperCAmelCase      	=		crop_size

           UpperCAmelCase      	=		input_size_patches
           UpperCAmelCase      	=		total_mask_patches
           UpperCAmelCase      	=		mask_group_max_patches
           UpperCAmelCase      	=		mask_group_min_patches
           UpperCAmelCase      	=		mask_group_min_aspect_ratio
           UpperCAmelCase      	=		mask_group_max_aspect_ratio

           UpperCAmelCase      	=		codebook_do_resize
           UpperCAmelCase      	=		codebook_size
           UpperCAmelCase      	=		codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
           UpperCAmelCase      	=		codebook_do_center_crop
           UpperCAmelCase      	=		codebook_crop_size
           UpperCAmelCase      	=		codebook_do_map_pixels
           UpperCAmelCase      	=		codebook_do_normalize
           UpperCAmelCase      	=		codebook_image_mean
           UpperCAmelCase      	=		codebook_image_std
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           return {
               "image_mean": self.image_mean,
               "image_std": self.image_std,
               "do_normalize": self.do_normalize,
               "do_resize": self.do_resize,
               "size": self.size,
               "resample": self.resample,
               "do_rescale": self.do_rescale,
               "rescale_factor": self.rescale_factor,
               "do_center_crop": self.do_center_crop,
               "crop_size": self.crop_size,
               "input_size_patches": self.input_size_patches,
               "total_mask_patches": self.total_mask_patches,
               "mask_group_max_patches": self.mask_group_max_patches,
               "mask_group_min_patches": self.mask_group_min_patches,
               "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
               "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
               "codebook_do_resize": self.codebook_do_resize,
               "codebook_size": self.codebook_size,
               "codebook_resample": self.codebook_resample,
               "codebook_do_center_crop": self.codebook_do_center_crop,
               "codebook_crop_size": self.codebook_crop_size,
               "codebook_do_map_pixels": self.codebook_do_map_pixels,
               "codebook_do_normalize": self.codebook_do_normalize,
               "codebook_image_mean": self.codebook_image_mean,
               "codebook_image_std": self.codebook_image_std,
           }
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           return (self.size["height"], self.size["width"])
       def    A				(     self						:     int    ):



           '''simple docstring'''

           return (
               (self.input_size_patches, self.input_size_patches)
               if not isinstance(self.input_size_patches       ,    lowercase_    )
               else self.input_size_patches
           )







       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           return (self.codebook_size["height"], self.codebook_size["width"])






@require_torch
@require_vision
class       _a  (     UpperCamelCase_							,							unittest.TestCase			):
       __a    :					Union[str, Any]           = FlavaImageProcessor if is_vision_available() else None
       __a    :					Optional[int]           = None
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		FlavaImageProcessingTester(self    )
       @property
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           return self.image_processor_tester.prepare_image_processor_dict()
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.image_processing_class(**self.image_processor_dict    )
           self.assertTrue(hasattr(lowercase_       ,    '''image_mean'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''image_std'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''do_normalize'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''do_resize'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''resample'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''crop_size'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''do_center_crop'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''do_rescale'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''rescale_factor'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''masking_generator'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_do_resize'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_size'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_resample'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_do_center_crop'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_crop_size'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_do_map_pixels'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_do_normalize'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_image_mean'''    )    )
           self.assertTrue(hasattr(lowercase_       ,    '''codebook_image_std'''    )    )
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.image_processing_class.from_dict(self.image_processor_dict    )
           self.assertEqual(image_processor.size       ,    {'''height''': 224, '''width''': 224}    )
           self.assertEqual(image_processor.crop_size       ,    {'''height''': 224, '''width''': 224}    )
           self.assertEqual(image_processor.codebook_size       ,    {'''height''': 112, '''width''': 112}    )
           self.assertEqual(image_processor.codebook_crop_size       ,    {'''height''': 112, '''width''': 112}    )

           UpperCAmelCase      	=		self.image_processing_class.from_dict(
               self.image_processor_dict       ,    size=42       ,    crop_size=84       ,    codebook_size=33       ,    codebook_crop_size=66    )
           self.assertEqual(image_processor.size       ,    {'''height''': 42, '''width''': 42}    )
           self.assertEqual(image_processor.crop_size       ,    {'''height''': 84, '''width''': 84}    )
           self.assertEqual(image_processor.codebook_size       ,    {'''height''': 33, '''width''': 33}    )
           self.assertEqual(image_processor.codebook_crop_size       ,    {'''height''': 66, '''width''': 66}    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           pass
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.image_processing_class(**self.image_processor_dict    )
           # create random PIL images
           UpperCAmelCase      	=		prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=lowercase_    )
           for image in image_inputs:
               self.assertIsInstance(lowercase_       ,    PIL.Image.Image    )

           # Test not batched input
           UpperCAmelCase      	=		image_processing(image_inputs[0]       ,    return_tensors='''pt'''    )

           # Test no bool masked pos
           self.assertFalse('''bool_masked_pos''' in encoded_images    )

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_image_size()

           self.assertEqual(
               encoded_images.pixel_values.shape       ,    (1, self.image_processor_tester.num_channels, expected_height, expected_width)       ,    )

           # Test batched
           UpperCAmelCase      	=		image_processing(lowercase_       ,    return_tensors='''pt'''    )
           UpperCAmelCase      	=		self.image_processor_tester.get_expected_image_size()

           # Test no bool masked pos
           self.assertFalse('''bool_masked_pos''' in encoded_images    )

           self.assertEqual(
               encoded_images.pixel_values.shape       ,    (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )       ,    )
       def    A				(     self						:     Optional[Any]       ,    lowercase						:     Any       ,    lowercase						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.image_processing_class(**self.image_processor_dict    )
           # create random tensors
           UpperCAmelCase      	=		prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=lowercase_       ,    **lowercase_    )
           for image in image_inputs:
               self.assertIsInstance(lowercase_       ,    lowercase_    )

           # Test not batched input
           UpperCAmelCase      	=		image_processing(image_inputs[0]       ,    return_tensors='''pt'''    )

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_image_size()
           self.assertEqual(
               encoded_images.pixel_values.shape       ,    (1, self.image_processor_tester.num_channels, expected_height, expected_width)       ,    )

           UpperCAmelCase      	=		image_processing(lowercase_       ,    return_image_mask=lowercase_       ,    return_tensors='''pt'''    )

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_image_size()
           self.assertEqual(
               encoded_images.pixel_values.shape       ,    (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )       ,    )

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_mask_size()
           self.assertEqual(
               encoded_images.bool_masked_pos.shape       ,    (
                   self.image_processor_tester.batch_size,
                   expected_height,
                   expected_width,
               )       ,    )

           # Test batched
           UpperCAmelCase      	=		image_processing(lowercase_       ,    return_tensors='''pt'''    ).pixel_values

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_image_size()
           self.assertEqual(
               encoded_images.shape       ,    (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )       ,    )

           # Test masking
           UpperCAmelCase      	=		image_processing(lowercase_       ,    return_image_mask=lowercase_       ,    return_tensors='''pt'''    )

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_image_size()
           self.assertEqual(
               encoded_images.pixel_values.shape       ,    (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )       ,    )

           UpperCAmelCase      	=		self.image_processor_tester.get_expected_mask_size()
           self.assertEqual(
               encoded_images.bool_masked_pos.shape       ,    (
                   self.image_processor_tester.batch_size,
                   expected_height,
                   expected_width,
               )       ,    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           self._test_call_framework(np.ndarray       ,    prepare_kwargs={'''numpify''': True}    )
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           self._test_call_framework(torch.Tensor       ,    prepare_kwargs={'''torchify''': True}    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           random.seed(1_234    )
           UpperCAmelCase      	=		self.image_processing_class(**self.image_processor_dict    )
           UpperCAmelCase      	=		prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=lowercase_       ,    torchify=lowercase_    )

           # Test not batched input
           UpperCAmelCase      	=		image_processing(image_inputs[0]       ,    return_image_mask=lowercase_       ,    return_tensors='''pt'''    )
           self.assertEqual(encoded_images.bool_masked_pos.sum().item()       ,    75    )







       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.image_processing_class(**self.image_processor_dict    )
           # create random PIL images
           UpperCAmelCase      	=		prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=lowercase_    )
           for image in image_inputs:
               self.assertIsInstance(lowercase_       ,    PIL.Image.Image    )

           # Test not batched input
           UpperCAmelCase      	=		image_processing(image_inputs[0]       ,    return_codebook_pixels=lowercase_       ,    return_tensors='''pt'''    )
           UpperCAmelCase      	=		self.image_processor_tester.get_expected_codebook_image_size()
           self.assertEqual(
               encoded_images.codebook_pixel_values.shape       ,    (1, self.image_processor_tester.num_channels, expected_height, expected_width)       ,    )

           # Test batched
           UpperCAmelCase      	=		image_processing(lowercase_       ,    return_codebook_pixels=lowercase_       ,    return_tensors='''pt'''    )
           UpperCAmelCase      	=		self.image_processor_tester.get_expected_codebook_image_size()
           self.assertEqual(
               encoded_images.codebook_pixel_values.shape       ,    (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )       ,    )











'''simple docstring'''







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       _a  (     __a							,							unittest.TestCase			):
       __a    :					Dict           = RealmTokenizer
       __a    :					Dict           = RealmTokenizerFast
       __a    :					Union[str, Any]           = True
       __a    :					Optional[int]           = True
       __a    :					int           = filter_non_english
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           super().setUp()

           UpperCAmelCase      	=		[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           UpperCAmelCase      	=		os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES['''vocab_file''']    )
           with open(self.vocab_file       ,    '''w'''       ,    encoding='''utf-8'''    ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
       def    A				(     self						:     int       ,    lowercase						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''
           UpperCAmelCase      	=		'''unwanted, running'''
           return input_text, output_text
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		tokenizer.tokenize('''UNwant\u00E9d,running'''    )
           self.assertListEqual(lowercase       ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )       ,    [9, 6, 7, 12, 10, 11]    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           if not self.test_rust_tokenizer:
               return

           UpperCAmelCase      	=		self.get_tokenizer()
           UpperCAmelCase      	=		self.get_rust_tokenizer()

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           # With lower casing
           UpperCAmelCase      	=		self.get_tokenizer(do_lower_case=lowercase    )
           UpperCAmelCase      	=		self.get_rust_tokenizer(do_lower_case=lowercase    )

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    )       ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''h\u00E9llo''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    never_split=['''[UNK]''']    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           UpperCAmelCase      	=		{}
           for i, token in enumerate(lowercase    ):
               UpperCAmelCase      	=		i
           UpperCAmelCase      	=		WordpieceTokenizer(vocab=lowercase       ,    unk_token='''[UNK]'''    )

           self.assertListEqual(tokenizer.tokenize(''''''    )       ,    []    )

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''    )       ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    )       ,    ['''[UNK]''', '''runn''', '''##ing''']    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           self.assertTrue(_is_whitespace(''' '''    )    )
           self.assertTrue(_is_whitespace('''\t'''    )    )
           self.assertTrue(_is_whitespace('''\r'''    )    )
           self.assertTrue(_is_whitespace('''\n'''    )    )
           self.assertTrue(_is_whitespace('''\u00A0'''    )    )

           self.assertFalse(_is_whitespace('''A'''    )    )
           self.assertFalse(_is_whitespace('''-'''    )    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           self.assertTrue(_is_control('''\u0005'''    )    )

           self.assertFalse(_is_control('''A'''    )    )
           self.assertFalse(_is_control(''' '''    )    )
           self.assertFalse(_is_control('''\t'''    )    )
           self.assertFalse(_is_control('''\r'''    )    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           self.assertTrue(_is_punctuation('''-'''    )    )
           self.assertTrue(_is_punctuation('''$'''    )    )
           self.assertTrue(_is_punctuation('''`'''    )    )
           self.assertTrue(_is_punctuation('''.'''    )    )

           self.assertFalse(_is_punctuation('''A'''    )    )
           self.assertFalse(_is_punctuation(''' '''    )    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )

           if self.test_rust_tokenizer:
               UpperCAmelCase      	=		self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )
       @slow
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class.from_pretrained('''bert-base-uncased'''    )

           UpperCAmelCase      	=		tokenizer.encode('''sequence builders'''       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		tokenizer.encode('''multi-sequence build'''       ,    add_special_tokens=lowercase    )

           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase    )
           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase       ,    lowercase    )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]
       def    A				(     self						:     int    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )

                   UpperCAmelCase      	=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                   UpperCAmelCase      	=		tokenizer_r.encode_plus(
                       lowercase       ,    return_attention_mask=lowercase       ,    return_token_type_ids=lowercase       ,    return_offsets_mapping=lowercase       ,    add_special_tokens=lowercase       ,    )

                   UpperCAmelCase      	=		tokenizer_r.do_lower_case if hasattr(lowercase       ,    '''do_lower_case'''    ) else False
                   UpperCAmelCase      	=		(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
                   self.assertEqual([e[0] for e in expected_results]       ,    tokens['''offset_mapping''']    )







       @slow
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		self.tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   UpperCAmelCase      	=		tokenizer_r.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )
                   UpperCAmelCase      	=		tokenizer_p.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )

                   UpperCAmelCase      	=		(2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape       ,    lowercase    )

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape       ,    lowercase    )




