




'''simple docstring'''







import argparse
import json
import os

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextImageProcessor, ConvNextVaConfig, ConvNextVaForImageClassification
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
A			   =logging.get_logger(__name__)





def 						snake_case_							(_a     :							Any					):
    UpperCAmelCase      	=		ConvNextVaConfig()

    if "atto" in checkpoint_url:
        UpperCAmelCase      	=		[2, 2, 6, 2]
        UpperCAmelCase      	=		[4_0, 8_0, 1_6_0, 3_2_0]
    if "femto" in checkpoint_url:
        UpperCAmelCase      	=		[2, 2, 6, 2]
        UpperCAmelCase      	=		[4_8, 9_6, 1_9_2, 3_8_4]
    if "pico" in checkpoint_url:
        UpperCAmelCase      	=		[2, 2, 6, 2]
        UpperCAmelCase      	=		[6_4, 1_2_8, 2_5_6, 5_1_2]
    if "nano" in checkpoint_url:
        UpperCAmelCase      	=		[2, 2, 8, 2]
        UpperCAmelCase      	=		[8_0, 1_6_0, 3_2_0, 6_4_0]
    if "tiny" in checkpoint_url:
        UpperCAmelCase      	=		[3, 3, 9, 3]
        UpperCAmelCase      	=		[9_6, 1_9_2, 3_8_4, 7_6_8]
    if "base" in checkpoint_url:
        UpperCAmelCase      	=		[3, 3, 2_7, 3]
        UpperCAmelCase      	=		[1_2_8, 2_5_6, 5_1_2, 1_0_2_4]
    if "large" in checkpoint_url:
        UpperCAmelCase      	=		[3, 3, 2_7, 3]
        UpperCAmelCase      	=		[1_9_2, 3_8_4, 7_6_8, 1_5_3_6]
    if "huge" in checkpoint_url:
        UpperCAmelCase      	=		[3, 3, 2_7, 3]
        UpperCAmelCase      	=		[3_5_2, 7_0_4, 1_4_0_8, 2_8_1_6]

    UpperCAmelCase      	=		1_0_0_0
    UpperCAmelCase      	=		'''imagenet-1k-id2label.json'''
    UpperCAmelCase      	=		(1, 1_0_0_0)

    UpperCAmelCase      	=		'''huggingface/label-files'''
    UpperCAmelCase      	=		num_labels
    UpperCAmelCase      	=		json.load(open(hf_hub_download(_a   ,       _a   ,       repo_type='''dataset'''					)   ,       '''r'''					)					)
    UpperCAmelCase      	=		{int(_a					): v for k, v in idalabel.items()}

    UpperCAmelCase      	=		idalabel
    UpperCAmelCase      	=		{v: k for k, v in idalabel.items()}
    UpperCAmelCase      	=		hidden_sizes
    UpperCAmelCase      	=		depths

    return config, expected_shape





def 						snake_case_							(_a     :							str					):
    if "downsample_layers.0.0" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.0.0'''   ,       '''embeddings.patch_embeddings'''					)
    if "downsample_layers.0.1" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.0.1'''   ,       '''embeddings.norm'''					)  # we rename to layernorm later on
    if "downsample_layers.1.0" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.1.0'''   ,       '''stages.1.downsampling_layer.0'''					)
    if "downsample_layers.1.1" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.1.1'''   ,       '''stages.1.downsampling_layer.1'''					)
    if "downsample_layers.2.0" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.2.0'''   ,       '''stages.2.downsampling_layer.0'''					)
    if "downsample_layers.2.1" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.2.1'''   ,       '''stages.2.downsampling_layer.1'''					)
    if "downsample_layers.3.0" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.3.0'''   ,       '''stages.3.downsampling_layer.0'''					)
    if "downsample_layers.3.1" in name:
        UpperCAmelCase      	=		name.replace('''downsample_layers.3.1'''   ,       '''stages.3.downsampling_layer.1'''					)
    if "stages" in name and "downsampling_layer" not in name:
        # stages.0.0. for instance should be renamed to stages.0.layers.0.
        UpperCAmelCase      	=		name[: len('''stages.0'''					)] + '''.layers''' + name[len('''stages.0'''					) :]
    if "gamma" in name:
        UpperCAmelCase      	=		name.replace('''gamma'''   ,       '''weight'''					)
    if "beta" in name:
        UpperCAmelCase      	=		name.replace('''beta'''   ,       '''bias'''					)
    if "stages" in name:
        UpperCAmelCase      	=		name.replace('''stages'''   ,       '''encoder.stages'''					)
    if "norm" in name:
        UpperCAmelCase      	=		name.replace('''norm'''   ,       '''layernorm'''					)
    if "head" in name:
        UpperCAmelCase      	=		name.replace('''head'''   ,       '''classifier'''					)

    return name





def 						snake_case_							():
    UpperCAmelCase      	=		'''http://images.cocodataset.org/val2017/000000039769.jpg'''
    UpperCAmelCase      	=		Image.open(requests.get(_a   ,       stream=_a					).raw					)
    return im





def 						snake_case_							(_a     :							Tuple					):
    if "224" in checkpoint_url:
        UpperCAmelCase      	=		2_2_4
        UpperCAmelCase      	=		2_2_4 / 2_5_6
    elif "384" in checkpoint_url:
        UpperCAmelCase      	=		3_8_4
        UpperCAmelCase      	=		None
    else:
        UpperCAmelCase      	=		5_1_2
        UpperCAmelCase      	=		None

    return ConvNextImageProcessor(
        size=_a   ,       crop_pct=_a   ,       image_mean=[0.485, 0.456, 0.406]   ,       image_std=[0.229, 0.224, 0.225]   ,       resample=PILImageResampling.BICUBIC   ,       )





@torch.no_grad()
def 						snake_case_							(_a     :							List[Any]   ,       _a     :							int   ,       _a     :							int   ,       _a     :							Any					):
    print('''Downloading original model from checkpoint...'''					)
    # define ConvNeXTV2 configuration based on URL
    UpperCAmelCase	,					UpperCAmelCase      	=		get_convnextva_config(_a					)
    # load original state_dict from URL
    UpperCAmelCase      	=		torch.hub.load_state_dict_from_url(_a					)['''model''']

    print('''Converting model parameters...'''					)
    # rename keys
    for key in state_dict.copy().keys():
        UpperCAmelCase      	=		state_dict.pop(_a					)
        UpperCAmelCase      	=		val
    # add prefix to all keys expect classifier head
    for key in state_dict.copy().keys():
        UpperCAmelCase      	=		state_dict.pop(_a					)
        if not key.startswith('''classifier'''					):
            UpperCAmelCase      	=		'''convnextv2.''' + key
        UpperCAmelCase      	=		val

    # load HuggingFace model
    UpperCAmelCase      	=		ConvNextVaForImageClassification(_a					)
    model.load_state_dict(_a					)
    model.eval()

    # Check outputs on an image, prepared by ConvNextImageProcessor
    UpperCAmelCase      	=		convert_preprocessor(_a					)
    UpperCAmelCase      	=		preprocessor(images=prepare_img()   ,       return_tensors='''pt'''					)
    UpperCAmelCase      	=		model(**_a					).logits

    # note: the logits below were obtained without center cropping
    if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.3930, 0.1747, -0.5246, 0.4177, 0.4295]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_femto_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.1727, -0.5341, -0.7818, -0.4745, -0.6566]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_pico_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.0333, 0.1563, -0.9137, 0.1054, 0.0381]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_nano_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.1744, -0.1555, -0.0713, 0.0950, -0.1431]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_tiny_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([0.9996, 0.1966, -0.4386, -0.3472, 0.6661]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_base_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.2553, -0.6708, -0.1359, 0.2518, -0.2488]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_large_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.0673, -0.5627, -0.3753, -0.2722, 0.0178]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_huge_1k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.6377, -0.7458, -0.2150, 0.1184, -0.0597]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([1.0799, 0.2322, -0.8860, 1.0219, 0.6231]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_384_ema.pt":
        UpperCAmelCase      	=		torch.tensor([0.3766, 0.4917, -1.1426, 0.9942, 0.6024]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([0.4220, -0.6919, -0.4317, -0.2881, -0.6609]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_384_ema.pt":
        UpperCAmelCase      	=		torch.tensor([0.1082, -0.8286, -0.5095, 0.4681, -0.8085]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.2419, -0.6221, 0.2176, -0.0980, -0.7527]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_384_ema.pt":
        UpperCAmelCase      	=		torch.tensor([0.0391, -0.4371, 0.3786, 0.1251, -0.2784]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_224_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.0504, 0.5636, -0.1729, -0.6507, -0.3949]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_384_ema.pt":
        UpperCAmelCase      	=		torch.tensor([0.3560, 0.9486, 0.3149, -0.2667, -0.5138]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_384_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.2469, -0.4550, -0.5853, -0.0810, 0.0309]					)
    elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_512_ema.pt":
        UpperCAmelCase      	=		torch.tensor([-0.3090, 0.0802, -0.0682, -0.1979, -0.2826]					)
    else:
        raise ValueError(F"Unknown URL: {checkpoint_url}"					)

    assert torch.allclose(logits[0, :5]   ,       _a   ,       atol=1E-3					)
    assert logits.shape == expected_shape
    print('''Model outputs match the original results!'''					)

    if save_model:
        print('''Saving model to local...'''					)
        # Create folder to save model
        if not os.path.isdir(_a					):
            os.mkdir(_a					)

        model.save_pretrained(_a					)
        preprocessor.save_pretrained(_a					)

    UpperCAmelCase      	=		'''convnextv2'''
    if "atto" in checkpoint_url:
        model_name += "-atto"
    if "femto" in checkpoint_url:
        model_name += "-femto"
    if "pico" in checkpoint_url:
        model_name += "-pico"
    if "nano" in checkpoint_url:
        model_name += "-nano"
    elif "tiny" in checkpoint_url:
        model_name += "-tiny"
    elif "base" in checkpoint_url:
        model_name += "-base"
    elif "large" in checkpoint_url:
        model_name += "-large"
    elif "huge" in checkpoint_url:
        model_name += "-huge"
    if "22k" in checkpoint_url and "1k" not in checkpoint_url:
        model_name += "-22k"
    elif "22k" in checkpoint_url and "1k" in checkpoint_url:
        model_name += "-22k-1k"
    elif "1k" in checkpoint_url:
        model_name += "-1k"
    if "224" in checkpoint_url:
        model_name += "-224"
    elif "384" in checkpoint_url:
        model_name += "-384"
    elif "512" in checkpoint_url:
        model_name += "-512"

    if push_to_hub:
        print(F"Pushing {model_name} to the hub..."					)
        model.push_to_hub(_a					)
        preprocessor.push_to_hub(_a					)


if __name__ == "__main__":
   A			   =argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--checkpoint_url',
       default='https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt',
       type=str,
       help='URL of the original ConvNeXTV2 checkpoint you\'d like to convert.',
   )
   parser.add_argument(
       '--pytorch_dump_folder_path',
       default='model',
       type=str,
       help='Path to the output PyTorch model directory.',
   )
   parser.add_argument('--save_model', action='store_true', help='Save model to local')
   parser.add_argument('--push_to_hub', action='store_true', help='Push model and image preprocessor to the hub')

   A			   =parser.parse_args()
   convert_convnextva_checkpoint(
       args.checkpoint_url, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub
   )










'''simple docstring'''







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       _a  (     __a							,							unittest.TestCase			):
       __a    :					Dict           = RealmTokenizer
       __a    :					Dict           = RealmTokenizerFast
       __a    :					Union[str, Any]           = True
       __a    :					Optional[int]           = True
       __a    :					int           = filter_non_english
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           super().setUp()

           UpperCAmelCase      	=		[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           UpperCAmelCase      	=		os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES['''vocab_file''']    )
           with open(self.vocab_file       ,    '''w'''       ,    encoding='''utf-8'''    ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
       def    A				(     self						:     int       ,    lowercase						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''
           UpperCAmelCase      	=		'''unwanted, running'''
           return input_text, output_text
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		tokenizer.tokenize('''UNwant\u00E9d,running'''    )
           self.assertListEqual(lowercase       ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']    )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )       ,    [9, 6, 7, 12, 10, 11]    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           if not self.test_rust_tokenizer:
               return

           UpperCAmelCase      	=		self.get_tokenizer()
           UpperCAmelCase      	=		self.get_rust_tokenizer()

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           # With lower casing
           UpperCAmelCase      	=		self.get_tokenizer(do_lower_case=lowercase    )
           UpperCAmelCase      	=		self.get_rust_tokenizer(do_lower_case=lowercase    )

           UpperCAmelCase      	=		'''UNwant\u00E9d,running'''

           UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.tokenize(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase       ,    add_special_tokens=lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )

           UpperCAmelCase      	=		self.get_rust_tokenizer()
           UpperCAmelCase      	=		tokenizer.encode(lowercase    )
           UpperCAmelCase      	=		rust_tokenizer.encode(lowercase    )
           self.assertListEqual(lowercase       ,    lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    )       ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''h\u00E9llo''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )       ,    ['''hello''']    )
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    strip_accents=lowercase    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )       ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		BasicTokenizer(do_lower_case=lowercase       ,    never_split=['''[UNK]''']    )

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    )       ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           UpperCAmelCase      	=		{}
           for i, token in enumerate(lowercase    ):
               UpperCAmelCase      	=		i
           UpperCAmelCase      	=		WordpieceTokenizer(vocab=lowercase       ,    unk_token='''[UNK]'''    )

           self.assertListEqual(tokenizer.tokenize(''''''    )       ,    []    )

           self.assertListEqual(tokenizer.tokenize('''unwanted running'''    )       ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

           self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    )       ,    ['''[UNK]''', '''runn''', '''##ing''']    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           self.assertTrue(_is_whitespace(''' '''    )    )
           self.assertTrue(_is_whitespace('''\t'''    )    )
           self.assertTrue(_is_whitespace('''\r'''    )    )
           self.assertTrue(_is_whitespace('''\n'''    )    )
           self.assertTrue(_is_whitespace('''\u00A0'''    )    )

           self.assertFalse(_is_whitespace('''A'''    )    )
           self.assertFalse(_is_whitespace('''-'''    )    )
       def    A				(     self						:     Optional[int]    ):



           '''simple docstring'''

           self.assertTrue(_is_control('''\u0005'''    )    )

           self.assertFalse(_is_control('''A'''    )    )
           self.assertFalse(_is_control(''' '''    )    )
           self.assertFalse(_is_control('''\t'''    )    )
           self.assertFalse(_is_control('''\r'''    )    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           self.assertTrue(_is_punctuation('''-'''    )    )
           self.assertTrue(_is_punctuation('''$'''    )    )
           self.assertTrue(_is_punctuation('''`'''    )    )
           self.assertTrue(_is_punctuation('''.'''    )    )

           self.assertFalse(_is_punctuation('''A'''    )    )
           self.assertFalse(_is_punctuation(''' '''    )    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )

           if self.test_rust_tokenizer:
               UpperCAmelCase      	=		self.get_rust_tokenizer()
               self.assertListEqual(
                   [rust_tokenizer.tokenize(lowercase    ) for t in ['''Test''', '''\xad''', '''test''']]       ,    [['''[UNK]'''], [], ['''[UNK]''']]    )
       @slow
       def    A				(     self						:     int    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class.from_pretrained('''bert-base-uncased'''    )

           UpperCAmelCase      	=		tokenizer.encode('''sequence builders'''       ,    add_special_tokens=lowercase    )
           UpperCAmelCase      	=		tokenizer.encode('''multi-sequence build'''       ,    add_special_tokens=lowercase    )

           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase    )
           UpperCAmelCase      	=		tokenizer.build_inputs_with_special_tokens(lowercase       ,    lowercase    )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]
       def    A				(     self						:     int    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )

                   UpperCAmelCase      	=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                   UpperCAmelCase      	=		tokenizer_r.encode_plus(
                       lowercase       ,    return_attention_mask=lowercase       ,    return_token_type_ids=lowercase       ,    return_offsets_mapping=lowercase       ,    add_special_tokens=lowercase       ,    )

                   UpperCAmelCase      	=		tokenizer_r.do_lower_case if hasattr(lowercase       ,    '''do_lower_case'''    ) else False
                   UpperCAmelCase      	=		(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]       ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
                   self.assertEqual([e[0] for e in expected_results]       ,    tokens['''offset_mapping''']    )







       @slow
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                   UpperCAmelCase      	=		self.rust_tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		self.tokenizer_class.from_pretrained(lowercase       ,    **lowercase    )
                   UpperCAmelCase      	=		[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   UpperCAmelCase      	=		tokenizer_r.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )
                   UpperCAmelCase      	=		tokenizer_p.batch_encode_candidates(lowercase       ,    max_length=10       ,    return_tensors='''np'''    )

                   UpperCAmelCase      	=		(2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape       ,    lowercase    )

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape       ,    lowercase    )
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape       ,    lowercase    )




