





"""simple docstring"""





import collections
import inspect
import unittest

from transformers import NatConfig
from transformers.testing_utils import require_natten, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import NatBackbone, NatForImageClassification, NatModel
    from transformers.models.nat.modeling_nat import NAT_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor





class       snake_case		:

      def __init__(      self       :Optional[Any]						,							_lowerCamelCase       :Optional[int]						,							_lowerCamelCase       :str=1_3						,							_lowerCamelCase       :str=6_4						,							_lowerCamelCase       :Optional[int]=4						,							_lowerCamelCase       :Any=3						,							_lowerCamelCase       :Tuple=1_6						,							_lowerCamelCase       :Tuple=[1, 2, 1]						,							_lowerCamelCase       :str=[2, 4, 8]						,							_lowerCamelCase       :Dict=3						,							_lowerCamelCase       :Optional[Any]=2.0						,							_lowerCamelCase       :Tuple=True						,							_lowerCamelCase       :Dict=0.0						,							_lowerCamelCase       :Tuple=0.0						,							_lowerCamelCase       :Union[str, Any]=0.1						,							_lowerCamelCase       :int="gelu"						,							_lowerCamelCase       :Optional[int]=True						,							_lowerCamelCase       :str=0.0_2						,							_lowerCamelCase       :int=1e-5						,							_lowerCamelCase       :List[Any]=True						,							_lowerCamelCase       :Any=None						,							_lowerCamelCase       :List[Any]=True						,							_lowerCamelCase       :List[Any]=1_0						,							_lowerCamelCase       :Any=["stage1", "stage2"]						,							_lowerCamelCase       :Optional[int]=[1, 2]						,							):
       __SCREAMING_SNAKE_CASE			:      Dict 							=	parent
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	batch_size
       __SCREAMING_SNAKE_CASE			:      str 							=	image_size
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	patch_size
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	num_channels
       __SCREAMING_SNAKE_CASE			:      Any 							=	embed_dim
       __SCREAMING_SNAKE_CASE			:      Any 							=	depths
       __SCREAMING_SNAKE_CASE			:      Any 							=	num_heads
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	kernel_size
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	mlp_ratio
       __SCREAMING_SNAKE_CASE			:      Any 							=	qkv_bias
       __SCREAMING_SNAKE_CASE			:      Any 							=	hidden_dropout_prob
       __SCREAMING_SNAKE_CASE			:      Any 							=	attention_probs_dropout_prob
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	drop_path_rate
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	hidden_act
       __SCREAMING_SNAKE_CASE			:      str 							=	patch_norm
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	layer_norm_eps
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	initializer_range
       __SCREAMING_SNAKE_CASE			:      Dict 							=	is_training
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	scope
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	use_labels
       __SCREAMING_SNAKE_CASE			:      str 							=	num_labels
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	out_features
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	out_indices

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	None
       if self.use_labels:
        __SCREAMING_SNAKE_CASE			:      int 							=	ids_tensor([self.batch_size]						,							self.num_labels						)

       __SCREAMING_SNAKE_CASE			:      Dict 							=	self.get_config()

       return config, pixel_values, labels

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       return NatConfig(
           num_labels=self.num_labels						,							image_size=self.image_size						,							patch_size=self.patch_size						,							num_channels=self.num_channels						,							embed_dim=self.embed_dim						,							depths=self.depths						,							num_heads=self.num_heads						,							kernel_size=self.kernel_size						,							mlp_ratio=self.mlp_ratio						,							qkv_bias=self.qkv_bias						,							hidden_dropout_prob=self.hidden_dropout_prob						,							attention_probs_dropout_prob=self.attention_probs_dropout_prob						,							drop_path_rate=self.drop_path_rate						,							hidden_act=self.hidden_act						,							patch_norm=self.patch_norm						,							layer_norm_eps=self.layer_norm_eps						,							initializer_range=self.initializer_range						,							out_features=self.out_features						,							out_indices=self.out_indices						,							)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Tuple						,							_lowerCamelCase       :Optional[int]						,							_lowerCamelCase       :List[str]						,							_lowerCamelCase       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	NatModel(config=_lowerCamelCase						)
       model.to(_lowerCamelCase						)
       model.eval()
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	model(_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      List[str] 							=	(config.image_size // config.patch_size) // (2 ** (len(config.depths						) - 1))
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	int(config.embed_dim * 2 ** (len(config.depths						) - 1)						)

       self.parent.assertEqual(
           result.last_hidden_state.shape						,							(self.batch_size, expected_height, expected_width, expected_dim)						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Any						,							_lowerCamelCase       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	NatForImageClassification(_lowerCamelCase						)
       model.to(_lowerCamelCase						)
       model.eval()
       __SCREAMING_SNAKE_CASE			:      str 							=	model(_lowerCamelCase						,							labels=_lowerCamelCase						)
       self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.num_labels)						)

       # test greyscale images
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	1
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	NatForImageClassification(_lowerCamelCase						)
       model.to(_lowerCamelCase						)
       model.eval()

       __SCREAMING_SNAKE_CASE			:      str 							=	floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	model(_lowerCamelCase						)
       self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.num_labels)						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						,							_lowerCamelCase       :Dict						,							_lowerCamelCase       :Optional[Any]						,							_lowerCamelCase       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	NatBackbone(config=_lowerCamelCase						)
       model.to(_lowerCamelCase						)
       model.eval()
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	model(_lowerCamelCase						)

       # verify hidden states
       self.parent.assertEqual(len(result.feature_maps						)						,							len(config.out_features						)						)
       self.parent.assertListEqual(list(result.feature_maps[0].shape						)						,							[self.batch_size, model.channels[0], 1_6, 1_6]						)

       # verify channels
       self.parent.assertEqual(len(model.channels						)						,							len(config.out_features						)						)

       # verify backbone works with out_features=None
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	None
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	NatBackbone(config=_lowerCamelCase						)
       model.to(_lowerCamelCase						)
       model.eval()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	model(_lowerCamelCase						)

       # verify feature maps
       self.parent.assertEqual(len(result.feature_maps						)						,							1						)
       self.parent.assertListEqual(list(result.feature_maps[0].shape						)						,							[self.batch_size, model.channels[-1], 4, 4]						)

       # verify channels
       self.parent.assertEqual(len(model.channels						)						,							1						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.prepare_config_and_inputs()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	config_and_inputs
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	{'''pixel_values''': pixel_values}
       return config, inputs_dict







@require_natten
@require_torch
class       snake_case		(       __UpperCAmelCase			,					__UpperCAmelCase			,					unittest.TestCase       ):
      lowerCamelCase__      =			(
          (
              NatModel,
              NatForImageClassification,
              NatBackbone,
          )
          if is_torch_available()
          else ()
      )
      lowerCamelCase__      =			(
          {'''feature-extraction''': NatModel, '''image-classification''': NatForImageClassification}
          if is_torch_available()
          else {}
      )
      lowerCamelCase__      =			False

      lowerCamelCase__      =			False
      lowerCamelCase__      =			False
      lowerCamelCase__      =			False
      lowerCamelCase__      =			False

      def 						SCREAMING_SNAKE_CASE_      (      self       :int						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	NatModelTester(self						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	ConfigTester(self						,							config_class=_lowerCamelCase						,							embed_dim=3_7						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						):
       self.create_and_test_config_common_properties()
       self.config_tester.create_and_test_config_to_json_string()
       self.config_tester.create_and_test_config_to_json_file()
       self.config_tester.create_and_test_config_from_and_save_pretrained()
       self.config_tester.create_and_test_config_with_num_labels()
       self.config_tester.check_config_can_be_init_without_params()
       self.config_tester.check_config_arguments_init()

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						):
       return

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      str 							=	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_backbone(*_lowerCamelCase						)

      @unittest.skip(reason='''Nat does not use inputs_embeds'''						)
      def 						SCREAMING_SNAKE_CASE_      (      self       :int						):
       pass

      @unittest.skip(reason='''Nat does not use feedforward chunking'''						)
      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      int 							=	self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      str 							=	model_class(_lowerCamelCase						)
        self.assertIsInstance(model.get_input_embeddings()						,							(nn.Module)						)
        __SCREAMING_SNAKE_CASE			:      int 							=	model.get_output_embeddings()
        self.assertTrue(x is None or isinstance(_lowerCamelCase						,							nn.Linear						)						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      str 							=	self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	model_class(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	inspect.signature(model.forward						)
        # signature.parameters is an OrderedDict => so arg_names order is deterministic
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[*signature.parameters.keys()]

        __SCREAMING_SNAKE_CASE			:      Dict 							=	['''pixel_values''']
        self.assertListEqual(arg_names[:1]						,							_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       self.skipTest('''Nat\'s attention operation is handled entirely by NATTEN.'''						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :List[Any]						,							_lowerCamelCase       :Optional[Any]						,							_lowerCamelCase       :Any						,							_lowerCamelCase       :Tuple						):
       __SCREAMING_SNAKE_CASE			:      str 							=	model_class(_lowerCamelCase						)
       model.to(_lowerCamelCase						)
       model.eval()

       with torch.no_grad():
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	model(**self._prepare_for_class(_lowerCamelCase						,							_lowerCamelCase						)						)

       __SCREAMING_SNAKE_CASE			:      Dict 							=	outputs.hidden_states

       __SCREAMING_SNAKE_CASE			:      Any 							=	getattr(
           self.model_tester						,							'''expected_num_hidden_layers'''						,							len(self.model_tester.depths						) + 1						)
       self.assertEqual(len(_lowerCamelCase						)						,							_lowerCamelCase						)

       # Nat has a different seq_length
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	(
           config.patch_size
           if isinstance(config.patch_size						,							collections.abc.Iterable						)
           else (config.patch_size, config.patch_size)
       )

       __SCREAMING_SNAKE_CASE			:      int 							=	image_size[0] // patch_size[0]
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	image_size[1] // patch_size[1]

       self.assertListEqual(
           list(hidden_states[0].shape[-3:]						)						,							[height, width, self.model_tester.embed_dim]						,							)

       if model_class.__name__ != "NatBackbone":
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	outputs.reshaped_hidden_states
        self.assertEqual(len(_lowerCamelCase						)						,							_lowerCamelCase						)

        __SCREAMING_SNAKE_CASE			:      Tuple 							=	reshaped_hidden_states[0].shape
        __SCREAMING_SNAKE_CASE			:      Dict 							=	(
            reshaped_hidden_states[0].view(_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						).permute(0						,							2						,							3						,							1						)
        )
        self.assertListEqual(
            list(reshaped_hidden_states.shape[-3:]						)						,							[height, width, self.model_tester.embed_dim]						,							)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						):
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.model_tester.prepare_config_and_inputs_for_common()

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	(
           self.model_tester.image_size
           if isinstance(self.model_tester.image_size						,							collections.abc.Iterable						)
           else (self.model_tester.image_size, self.model_tester.image_size)
       )

       for model_class in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	True
        self.check_hidden_states_output(_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						)

        # check that output_hidden_states also work using config
        del inputs_dict["output_hidden_states"]
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	True

        self.check_hidden_states_output(_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						)

      @slow
      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       for model_name in NAT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        __SCREAMING_SNAKE_CASE			:      str 							=	NatModel.from_pretrained(_lowerCamelCase						)
        self.assertIsNotNone(_lowerCamelCase						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Dict 							=	self.model_tester.prepare_config_and_inputs_for_common()

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	_config_zero_init(_lowerCamelCase						)
       for model_class in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      int 							=	model_class(config=_lowerCamelCase						)
        for name, param in model.named_parameters():
         if "embeddings" not in name and param.requires_grad:
          self.assertIn(
              ((param.data.mean() * 1e9).round() / 1e9).item()						,							[0.0, 1.0]						,							msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''						,							)







@require_natten
@require_vision
@require_torch
class       snake_case		(       unittest.TestCase       ):

      @cached_property
      def 						SCREAMING_SNAKE_CASE_      (      self       :Tuple						):
       return AutoImageProcessor.from_pretrained('''shi-labs/nat-mini-in1k-224'''						) if is_vision_available() else None



      @slow
      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	NatForImageClassification.from_pretrained('''shi-labs/nat-mini-in1k-224'''						).to(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	self.default_image_processor

       __SCREAMING_SNAKE_CASE			:      int 							=	Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''						)
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	image_processor(images=_lowerCamelCase						,							return_tensors='''pt'''						).to(_lowerCamelCase						)

       # forward pass
       with torch.no_grad():
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	model(**_lowerCamelCase						)

       # verify the logits
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	torch.Size((1, 1_0_0_0)						)
       self.assertEqual(outputs.logits.shape						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	torch.tensor([0.3_8_0_5, -0.8_6_7_6, -0.3_9_1_2]						).to(_lowerCamelCase						)
       self.assertTrue(torch.allclose(outputs.logits[0, :3]						,							_lowerCamelCase						,							atol=1e-4						)						)







@require_torch
@require_natten
class       snake_case		(       unittest.TestCase			,					__UpperCAmelCase       ):
      lowerCamelCase__      =			(NatBackbone,) if is_torch_available() else ()
      lowerCamelCase__      =			NatConfig

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						):
       __SCREAMING_SNAKE_CASE			:      int 							=	NatModelTester(self						)








"""simple docstring"""





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       snake_case		(       __UpperCAmelCase			,					unittest.TestCase       ):
      lowerCamelCase__      =			PreTrainedTokenizerFast
      lowerCamelCase__      =			False
      lowerCamelCase__      =			True
      lowerCamelCase__      =			'''tokenizer_file'''

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	True

       __SCREAMING_SNAKE_CASE			:      int 							=	['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       __SCREAMING_SNAKE_CASE			:      str 							=	PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
       tokenizer.save_pretrained(self.tmpdirname						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      int 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	tmpdirname_orig

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      Dict 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer_with_special_tokens_change()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tmpdirname_orig



      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       __SCREAMING_SNAKE_CASE			:      str 							=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	('''a''' for _ in range(1_0_0_0						))
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase						,							length=1_0_0_0						,							vocab_size=5_0						)

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	new_tokenizer.encode('''a🤗'''						)
       self.assertEqual(_lowerCamelCase						,							[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained('''bert-base-cased'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	json.loads(tokenizer._tokenizer.to_str()						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	len(_lowerCamelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	['''tokenizer.4.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        json.dump(_lowerCamelCase						,							open(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							'''w'''						)						)

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        __SCREAMING_SNAKE_CASE			:      Dict 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						) + 1						)
        __SCREAMING_SNAKE_CASE			:      Dict 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							os.path.join(_lowerCamelCase						,							'''tokenizer.42.0.0.json'''						)						)
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	['''tokenizer.42.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						)						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	'''hf-internal-testing/test-two-tokenizers'''

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       __SCREAMING_SNAKE_CASE			:      int 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_7						)
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	json.loads(tokenizer._tokenizer.to_str()						)
       self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       __SCREAMING_SNAKE_CASE			:      Dict 							=	'''3.0.0'''
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_6						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	json.loads(old_tokenizer._tokenizer.to_str()						)
       self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	'''The Matrix is a 1999 science fiction action film.'''

       with concurrent.futures.ThreadPoolExecutor() as executor:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	[executor.submit(self.fetch						,							_lowerCamelCase						,							_lowerCamelCase						) for i in range(1_0						)]
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	[future.result() for future in futures]
        self.assertEqual(_lowerCamelCase						,							[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0						)]						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Optional[int]						):
       return tokenizer.encode(_lowerCamelCase						,							truncation='''longest_first'''						,							padding='''longest'''						)

