





"""simple docstring"""





import argparse

import requests
import torch
from PIL import Image

from transformers import (
    CLIPSegConfig,
    CLIPSegForImageSegmentation,
    CLIPSegProcessor,
    CLIPSegTextConfig,
    CLIPSegVisionConfig,
    CLIPTokenizer,
    ViTImageProcessor,
)




def 		lowerCAmelCase_     (  lowercase_       :							List[Any]		):
 __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	CLIPSegTextConfig()
 __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	CLIPSegVisionConfig(patch_size=16		)

 __SCREAMING_SNAKE_CASE			:      Dict 							=	True if '''refined''' in model_name else False
 __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	16 if '''rd16''' in model_name else 64

 __SCREAMING_SNAKE_CASE			:      List[str] 							=	CLIPSegConfig.from_text_vision_configs(
     lowercase_			,				lowercase_			,				use_complex_transposed_convolution=lowercase_			,				reduce_dim=lowercase_			,				)
 return config




def 		lowerCAmelCase_     (  lowercase_       :							Union[str, Any]		):
 # update prefixes
 if "clip_model" in name:
  __SCREAMING_SNAKE_CASE			:      Dict 							=	name.replace('''clip_model'''			,				'''clip'''		)
 if "transformer" in name:
  if "visual" in name:
   __SCREAMING_SNAKE_CASE			:      Any 							=	name.replace('''visual.transformer'''			,				'''vision_model'''		)
  else:
   __SCREAMING_SNAKE_CASE			:      int 							=	name.replace('''transformer'''			,				'''text_model'''		)
 if "resblocks" in name:
  __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	name.replace('''resblocks'''			,				'''encoder.layers'''		)
 if "ln_1" in name:
  __SCREAMING_SNAKE_CASE			:      List[Any] 							=	name.replace('''ln_1'''			,				'''layer_norm1'''		)
 if "ln_2" in name:
  __SCREAMING_SNAKE_CASE			:      List[Any] 							=	name.replace('''ln_2'''			,				'''layer_norm2'''		)
 if "c_fc" in name:
  __SCREAMING_SNAKE_CASE			:      str 							=	name.replace('''c_fc'''			,				'''fc1'''		)
 if "c_proj" in name:
  __SCREAMING_SNAKE_CASE			:      List[Any] 							=	name.replace('''c_proj'''			,				'''fc2'''		)
 if "attn" in name and "self" not in name:
  __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	name.replace('''attn'''			,				'''self_attn'''		)
 # text encoder
 if "token_embedding" in name:
  __SCREAMING_SNAKE_CASE			:      str 							=	name.replace('''token_embedding'''			,				'''text_model.embeddings.token_embedding'''		)
 if "positional_embedding" in name and "visual" not in name:
  __SCREAMING_SNAKE_CASE			:      Dict 							=	name.replace('''positional_embedding'''			,				'''text_model.embeddings.position_embedding.weight'''		)
 if "ln_final" in name:
  __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	name.replace('''ln_final'''			,				'''text_model.final_layer_norm'''		)
 # vision encoder
 if "visual.class_embedding" in name:
  __SCREAMING_SNAKE_CASE			:      int 							=	name.replace('''visual.class_embedding'''			,				'''vision_model.embeddings.class_embedding'''		)
 if "visual.conv1" in name:
  __SCREAMING_SNAKE_CASE			:      List[str] 							=	name.replace('''visual.conv1'''			,				'''vision_model.embeddings.patch_embedding'''		)
 if "visual.positional_embedding" in name:
  __SCREAMING_SNAKE_CASE			:      int 							=	name.replace('''visual.positional_embedding'''			,				'''vision_model.embeddings.position_embedding.weight'''		)
 if "visual.ln_pre" in name:
  __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	name.replace('''visual.ln_pre'''			,				'''vision_model.pre_layrnorm'''		)
 if "visual.ln_post" in name:
  __SCREAMING_SNAKE_CASE			:      int 							=	name.replace('''visual.ln_post'''			,				'''vision_model.post_layernorm'''		)
 # projection layers
 if "visual.proj" in name:
  __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	name.replace('''visual.proj'''			,				'''visual_projection.weight'''		)
 if "text_projection" in name:
  __SCREAMING_SNAKE_CASE			:      str 							=	name.replace('''text_projection'''			,				'''text_projection.weight'''		)
 # decoder
 if "trans_conv" in name:
  __SCREAMING_SNAKE_CASE			:      Dict 							=	name.replace('''trans_conv'''			,				'''transposed_convolution'''		)
 if "film_mul" in name or "film_add" in name or "reduce" in name or "transposed_convolution" in name:
  __SCREAMING_SNAKE_CASE			:      Tuple 							=	'''decoder.''' + name
 if "blocks" in name:
  __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	name.replace('''blocks'''			,				'''decoder.layers'''		)
 if "linear1" in name:
  __SCREAMING_SNAKE_CASE			:      List[Any] 							=	name.replace('''linear1'''			,				'''mlp.fc1'''		)
 if "linear2" in name:
  __SCREAMING_SNAKE_CASE			:      str 							=	name.replace('''linear2'''			,				'''mlp.fc2'''		)
 if "norm1" in name and "layer_" not in name:
  __SCREAMING_SNAKE_CASE			:      str 							=	name.replace('''norm1'''			,				'''layer_norm1'''		)
 if "norm2" in name and "layer_" not in name:
  __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	name.replace('''norm2'''			,				'''layer_norm2'''		)

 return name




def 		lowerCAmelCase_     (  lowercase_       :							int			,				lowercase_       :							Optional[int]		):
 for key in orig_state_dict.copy().keys():
  __SCREAMING_SNAKE_CASE			:      List[str] 							=	orig_state_dict.pop(lowercase_		)

  if key.startswith('''clip_model'''		) and "attn.in_proj" in key:
   __SCREAMING_SNAKE_CASE			:      Any 							=	key.split('''.'''		)
   if "visual" in key:
    __SCREAMING_SNAKE_CASE			:      List[str] 							=	int(key_split[4]		)
    __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	config.vision_config.hidden_size
    __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	'''vision_model'''
   else:
    __SCREAMING_SNAKE_CASE			:      Dict 							=	int(key_split[3]		)
    __SCREAMING_SNAKE_CASE			:      str 							=	config.text_config.hidden_size
    __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	'''text_model'''

   if "weight" in key:
    __SCREAMING_SNAKE_CASE			:      Any 							=	val[:dim, :]
    __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	val[
        dim : dim * 2, :
    ]
    __SCREAMING_SNAKE_CASE			:      List[Any] 							=	val[-dim:, :]
   else:
    __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	val[:dim]
    __SCREAMING_SNAKE_CASE			:      List[str] 							=	val[dim : dim * 2]
    __SCREAMING_SNAKE_CASE			:      int 							=	val[-dim:]
  elif "self_attn" in key and "out_proj" not in key:
   __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	key.split('''.'''		)
   __SCREAMING_SNAKE_CASE			:      Any 							=	int(key_split[1]		)
   __SCREAMING_SNAKE_CASE			:      str 							=	config.reduce_dim
   if "weight" in key:
    __SCREAMING_SNAKE_CASE			:      int 							=	val[:dim, :]
    __SCREAMING_SNAKE_CASE			:      str 							=	val[dim : dim * 2, :]
    __SCREAMING_SNAKE_CASE			:      str 							=	val[-dim:, :]
   else:
    __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	val[:dim]
    __SCREAMING_SNAKE_CASE			:      str 							=	val[dim : dim * 2]
    __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	val[-dim:]
  else:
   __SCREAMING_SNAKE_CASE			:      List[str] 							=	rename_key(lowercase_		)
   if "visual_projection" in new_name or "text_projection" in new_name:
    __SCREAMING_SNAKE_CASE			:      List[str] 							=	val.T
   __SCREAMING_SNAKE_CASE			:      Dict 							=	val

 return orig_state_dict




def 		lowerCAmelCase_     (  ):
 __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	'''http://images.cocodataset.org/val2017/000000039769.jpg'''
 __SCREAMING_SNAKE_CASE			:      Any 							=	Image.open(requests.get(lowercase_			,				stream=lowercase_		).raw		)
 return image




def 		lowerCAmelCase_     (  lowercase_       :							Dict			,				lowercase_       :							List[Any]			,				lowercase_       :							Optional[int]			,				lowercase_       :							Union[str, Any]		):
 __SCREAMING_SNAKE_CASE			:      List[Any] 							=	get_clipseg_config(lowercase_		)
 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	CLIPSegForImageSegmentation(lowercase_		)
 model.eval()

 __SCREAMING_SNAKE_CASE			:      int 							=	torch.load(lowercase_			,				map_location='''cpu'''		)

 # remove some keys
 for key in state_dict.copy().keys():
  if key.startswith('''model'''		):
   state_dict.pop(lowercase_			,				lowercase_		)

  # rename some keys
 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	convert_state_dict(lowercase_			,				lowercase_		)
 __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	model.load_state_dict(lowercase_			,				strict=lowercase_		)

 if missing_keys != ["clip.text_model.embeddings.position_ids", "clip.vision_model.embeddings.position_ids"]:
  raise ValueError('''Missing keys that are not expected: {}'''.format(lowercase_		)		)
 if unexpected_keys != ["decoder.reduce.weight", "decoder.reduce.bias"]:
  raise ValueError(F'''Unexpected keys: {unexpected_keys}'''		)

 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	ViTImageProcessor(size=352		)
 __SCREAMING_SNAKE_CASE			:      str 							=	CLIPTokenizer.from_pretrained('''openai/clip-vit-base-patch32'''		)
 __SCREAMING_SNAKE_CASE			:      Tuple 							=	CLIPSegProcessor(image_processor=lowercase_			,				tokenizer=lowercase_		)

 __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	prepare_img()
 __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	['''a glass''', '''something to fill''', '''wood''', '''a jar''']

 __SCREAMING_SNAKE_CASE			:      List[Any] 							=	processor(text=lowercase_			,				images=[image] * len(lowercase_		)			,				padding='''max_length'''			,				return_tensors='''pt'''		)

 with torch.no_grad():
  __SCREAMING_SNAKE_CASE			:      List[str] 							=	model(**lowercase_		)

 # verify values
 __SCREAMING_SNAKE_CASE			:      List[Any] 							=	torch.tensor([0.1_110, -0.1_882, 0.1_645]		)
 __SCREAMING_SNAKE_CASE			:      Tuple 							=	torch.tensor([0.2_692, -0.7_197, -0.1_328]		)
 if model_name == "clipseg-rd64-refined":
  __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	torch.tensor(
      [[-10.0_407, -9.9_431, -10.2_646], [-9.9_751, -9.7_064, -9.9_586], [-9.6_891, -9.5_645, -9.9_618]]		)
 elif model_name == "clipseg-rd64":
  __SCREAMING_SNAKE_CASE			:      str 							=	torch.tensor(
      [[-7.2_877, -7.2_711, -7.2_463], [-7.2_652, -7.2_780, -7.2_520], [-7.2_239, -7.2_204, -7.2_001]]		)
 elif model_name == "clipseg-rd16":
  __SCREAMING_SNAKE_CASE			:      Tuple 							=	torch.tensor(
      [[-6.3_955, -6.4_055, -6.4_151], [-6.3_911, -6.4_033, -6.4_100], [-6.3_474, -6.3_702, -6.3_762]]		)
 else:
  raise ValueError(F'''Model name {model_name} not supported.'''		)

 assert torch.allclose(outputs.logits[0, :3, :3]			,				lowercase_			,				atol=1E-3		)
 assert torch.allclose(outputs.conditional_embeddings[0, :3]			,				lowercase_			,				atol=1E-3		)
 assert torch.allclose(outputs.pooled_output[0, :3]			,				lowercase_			,				atol=1E-3		)
 print('''Looks ok!'''		)

 if pytorch_dump_folder_path is not None:
  print(F'''Saving model and processor to {pytorch_dump_folder_path}'''		)
  model.save_pretrained(lowercase_		)
  processor.save_pretrained(lowercase_		)

 if push_to_hub:
  print(F'''Pushing model and processor for {model_name} to the hub'''		)
  model.push_to_hub(F'''CIDAS/{model_name}'''		)
  processor.push_to_hub(F'''CIDAS/{model_name}'''		)


if __name__ == "__main__":
    _lowerCamelCase  					=     argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        '''--model_name''',
        default='''clipseg-rd64''',
        type=str,
        choices=['''clipseg-rd16''', '''clipseg-rd64''', '''clipseg-rd64-refined'''],
        help=(
            '''Name of the model. Supported models are: clipseg-rd64, clipseg-rd16 and clipseg-rd64-refined (rd meaning'''
            ''' reduce dimension)'''
        ),
    )
    parser.add_argument(
        '''--checkpoint_path''',
        default='''/Users/nielsrogge/Documents/CLIPSeg/clip_plus_rd64-uni.pth''',
        type=str,
        help=(
            '''Path to the original checkpoint. Note that the script assumes that the checkpoint includes both CLIP and'''
            ''' the decoder weights.'''
        ),
    )
    parser.add_argument(
        '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
    )
    parser.add_argument(
        '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
    )

    _lowerCamelCase  					=     parser.parse_args()
    convert_clipseg_checkpoint(args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub)








"""simple docstring"""





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       snake_case		(       __UpperCAmelCase			,					unittest.TestCase       ):
      lowerCamelCase__      =			PreTrainedTokenizerFast
      lowerCamelCase__      =			False
      lowerCamelCase__      =			True
      lowerCamelCase__      =			'''tokenizer_file'''

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	True

       __SCREAMING_SNAKE_CASE			:      int 							=	['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       __SCREAMING_SNAKE_CASE			:      str 							=	PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
       tokenizer.save_pretrained(self.tmpdirname						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      int 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	tmpdirname_orig

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      Dict 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer_with_special_tokens_change()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tmpdirname_orig



      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       __SCREAMING_SNAKE_CASE			:      str 							=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	('''a''' for _ in range(1_0_0_0						))
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase						,							length=1_0_0_0						,							vocab_size=5_0						)

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	new_tokenizer.encode('''a🤗'''						)
       self.assertEqual(_lowerCamelCase						,							[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained('''bert-base-cased'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	json.loads(tokenizer._tokenizer.to_str()						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	len(_lowerCamelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	['''tokenizer.4.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        json.dump(_lowerCamelCase						,							open(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							'''w'''						)						)

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        __SCREAMING_SNAKE_CASE			:      Dict 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						) + 1						)
        __SCREAMING_SNAKE_CASE			:      Dict 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							os.path.join(_lowerCamelCase						,							'''tokenizer.42.0.0.json'''						)						)
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	['''tokenizer.42.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						)						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	'''hf-internal-testing/test-two-tokenizers'''

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       __SCREAMING_SNAKE_CASE			:      int 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_7						)
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	json.loads(tokenizer._tokenizer.to_str()						)
       self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       __SCREAMING_SNAKE_CASE			:      Dict 							=	'''3.0.0'''
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_6						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	json.loads(old_tokenizer._tokenizer.to_str()						)
       self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	'''The Matrix is a 1999 science fiction action film.'''

       with concurrent.futures.ThreadPoolExecutor() as executor:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	[executor.submit(self.fetch						,							_lowerCamelCase						,							_lowerCamelCase						) for i in range(1_0						)]
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	[future.result() for future in futures]
        self.assertEqual(_lowerCamelCase						,							[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0						)]						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Optional[int]						):
       return tokenizer.encode(_lowerCamelCase						,							truncation='''longest_first'''						,							padding='''longest'''						)

