





"""simple docstring"""





import tempfile
import unittest

import transformers
from transformers import XGLMConfig, XGLMTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
    import jax
    import jax.numpy as jnp
    import numpy as np

    from transformers.modeling_flax_pytorch_utils import (
        convert_pytorch_state_dict_to_flax,
        load_flax_weights_in_pytorch_model,
    )
    from transformers.models.xglm.modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel


if is_torch_available():
    import torch





@require_flax
class       snake_case		:

      def __init__(      self       :str						,							_lowerCamelCase       :int						,							_lowerCamelCase       :str=1_4						,							_lowerCamelCase       :Optional[int]=7						,							_lowerCamelCase       :List[str]=True						,							_lowerCamelCase       :Optional[int]=True						,							_lowerCamelCase       :Any=True						,							_lowerCamelCase       :Any=9_9						,							_lowerCamelCase       :Dict=3_2						,							_lowerCamelCase       :Any=5						,							_lowerCamelCase       :Dict=4						,							_lowerCamelCase       :List[Any]=3_7						,							_lowerCamelCase       :Dict="gelu"						,							_lowerCamelCase       :Optional[Any]=0.1						,							_lowerCamelCase       :int=0.1						,							_lowerCamelCase       :Union[str, Any]=5_1_2						,							_lowerCamelCase       :Tuple=0.0_2						,							_lowerCamelCase       :List[str]=None						,							):
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	parent
       __SCREAMING_SNAKE_CASE			:      Dict 							=	batch_size
       __SCREAMING_SNAKE_CASE			:      str 							=	seq_length
       __SCREAMING_SNAKE_CASE			:      str 							=	is_training
       __SCREAMING_SNAKE_CASE			:      Dict 							=	use_input_mask
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	use_labels
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	vocab_size
       __SCREAMING_SNAKE_CASE			:      Dict 							=	d_model
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	num_hidden_layers
       __SCREAMING_SNAKE_CASE			:      Any 							=	num_attention_heads
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	ffn_dim
       __SCREAMING_SNAKE_CASE			:      int 							=	activation_function
       __SCREAMING_SNAKE_CASE			:      Dict 							=	activation_dropout
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	attention_dropout
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	max_position_embeddings
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	initializer_range
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	None
       __SCREAMING_SNAKE_CASE			:      Dict 							=	0
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	2
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	1

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	np.clip(ids_tensor([self.batch_size, self.seq_length]						,							self.vocab_size						)						,							3						,							self.vocab_size						)

       __SCREAMING_SNAKE_CASE			:      Dict 							=	None
       if self.use_input_mask:
        __SCREAMING_SNAKE_CASE			:      str 							=	random_attention_mask([self.batch_size, self.seq_length]						)

       __SCREAMING_SNAKE_CASE			:      str 							=	XGLMConfig(
           vocab_size=self.vocab_size						,							d_model=self.hidden_size						,							num_layers=self.num_hidden_layers						,							attention_heads=self.num_attention_heads						,							ffn_dim=self.ffn_dim						,							activation_function=self.activation_function						,							activation_dropout=self.activation_dropout						,							attention_dropout=self.attention_dropout						,							max_position_embeddings=self.max_position_embeddings						,							initializer_range=self.initializer_range						,							use_cache=__A						,							bos_token_id=self.bos_token_id						,							eos_token_id=self.eos_token_id						,							pad_token_id=self.pad_token_id						,							)

       return (config, input_ids, input_mask)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.prepare_config_and_inputs()
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	config_and_inputs
       __SCREAMING_SNAKE_CASE			:      int 							=	{"input_ids": input_ids, "attention_mask": attention_mask}
       return config, inputs_dict

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.prepare_config_and_inputs()

       __SCREAMING_SNAKE_CASE			:      Dict 							=	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	ids_tensor([self.batch_size, self.seq_length]						,							vocab_size=2						)

       return (
           config,
           input_ids,
           attention_mask,
           encoder_hidden_states,
           encoder_attention_mask,
       )

      def 						SCREAMING_SNAKE_CASE_      (      self       :int						,							_lowerCamelCase       :Tuple						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Dict						,							_lowerCamelCase       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	2_0
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	model_class_name(__A						)

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	model.init_cache(input_ids.shape[0]						,							__A						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	jnp.ones((input_ids.shape[0], max_decoder_length)						,							dtype='''i4'''						)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1						)[None, :]						,							(input_ids.shape[0], input_ids.shape[-1] - 1)						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	model(
           input_ids[:, :-1]						,							attention_mask=__A						,							past_key_values=__A						,							position_ids=__A						,							)

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]						,							dtype='''i4'''						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	model(
           input_ids[:, -1:]						,							attention_mask=__A						,							past_key_values=outputs_cache.past_key_values						,							position_ids=__A						,							)

       __SCREAMING_SNAKE_CASE			:      int 							=	model(__A						)

       __SCREAMING_SNAKE_CASE			:      List[str] 							=	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
       self.parent.assertTrue(diff < 1e-3						,							msg=f'''Max diff is {diff}'''						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :Any						,							_lowerCamelCase       :Optional[int]						,							_lowerCamelCase       :Optional[int]						,							_lowerCamelCase       :Dict						):
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	2_0
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	model_class_name(__A						)

       __SCREAMING_SNAKE_CASE			:      Any 							=	jnp.concatenate(
           [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])						)]						,							axis=-1						,							)

       __SCREAMING_SNAKE_CASE			:      int 							=	model.init_cache(input_ids.shape[0]						,							__A						)
       __SCREAMING_SNAKE_CASE			:      int 							=	jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1						)[None, :]						,							(input_ids.shape[0], input_ids.shape[-1] - 1)						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	model(
           input_ids[:, :-1]						,							attention_mask=__A						,							past_key_values=__A						,							position_ids=__A						,							)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]						,							dtype='''i4'''						)
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	model(
           input_ids[:, -1:]						,							past_key_values=outputs_cache.past_key_values						,							attention_mask=__A						,							position_ids=__A						,							)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	model(__A						,							attention_mask=__A						)
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
       self.parent.assertTrue(diff < 1e-3						,							msg=f'''Max diff is {diff}'''						)







@require_sentencepiece
@require_flax
class       snake_case		(       __UpperCAmelCase			,					__UpperCAmelCase			,					unittest.TestCase       ):
      lowerCamelCase__      =			(FlaxXGLMModel, FlaxXGLMForCausalLM) if is_flax_available() else ()
      lowerCamelCase__      =			(FlaxXGLMForCausalLM,) if is_flax_available() else ()

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      int 							=	FlaxXGLMModelTester(self						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):





       for model_class_name in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_use_cache_forward(__A						,							__A						,							__A						,							__A						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						):





       for model_class_name in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_use_cache_forward_with_attn_mask(
            __A						,							__A						,							__A						,							__A						)

      @slow
      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	XGLMTokenizer.from_pretrained('''XGLM'''						,							padding_side='''left'''						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	tokenizer(['''Hello this is a long string''', '''Hey''']						,							return_tensors='''np'''						,							padding=__A						,							truncation=__A						)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	FlaxXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''						)
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	1
       __SCREAMING_SNAKE_CASE			:      int 							=	False

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	jax.jit(model.generate						)

       __SCREAMING_SNAKE_CASE			:      str 							=	jit_generate(inputs['''input_ids''']						,							attention_mask=inputs['''attention_mask''']						).sequences

       __SCREAMING_SNAKE_CASE			:      int 							=	tokenizer.batch_decode(__A						,							skip_special_tokens=__A						)

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	[
           "Hello this is a long string of questions, but I'm not sure if I'm",
           "Hey, I'm a newbie to the forum and I'",
       ]

       self.assertListEqual(__A						,							__A						)

      @is_pt_flax_cross_test
      def 						SCREAMING_SNAKE_CASE_      (      self       :int						):
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
        with self.subTest(model_class.__name__						):
         # prepare inputs
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	self._prepare_for_class(__A						,							__A						)
         __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	{k: torch.tensor(v.tolist()						) for k, v in prepared_inputs_dict.items()}

         # load corresponding PyTorch class
         __SCREAMING_SNAKE_CASE			:      int 							=	model_class.__name__[4:]  # Skip the "Flax" at the beginning
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	getattr(__A						,							__A						)

         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	pt_inputs["input_ids"].shape
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	np.random.randint(0						,							seq_length - 1						,							size=(batch_size,)						)
         for batch_idx, start_index in enumerate(__A						):
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	0
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	1
          __SCREAMING_SNAKE_CASE			:      Dict 							=	0
          __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	1
         __SCREAMING_SNAKE_CASE			:      int 							=	pt_model_class(__A						).eval()
         # Flax models don't use the `use_cache` option and cache is not returned as a default.
         # So we disable `use_cache` here for PyTorch model.
         __SCREAMING_SNAKE_CASE			:      Any 							=	False
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	model_class(__A						,							dtype=jnp.floataa						)
         __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	convert_pytorch_state_dict_to_flax(pt_model.state_dict()						,							__A						)
         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	fx_state

         with torch.no_grad():
          __SCREAMING_SNAKE_CASE			:      Any 							=	pt_model(**__A						).to_tuple()

         __SCREAMING_SNAKE_CASE			:      Tuple 							=	fx_model(**__A						).to_tuple()
         self.assertEqual(len(__A						)						,							len(__A						)						,							'''Output lengths differ between Flax and PyTorch'''						)
         for fx_output, pt_output in zip(__A						,							__A						):
          self.assert_almost_equals(fx_output[:, -1]						,							pt_output[:, -1].numpy()						,							4e-2						)

         with tempfile.TemporaryDirectory() as tmpdirname:
          pt_model.save_pretrained(__A						)
          __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	model_class.from_pretrained(__A						,							from_pt=__A						)

         __SCREAMING_SNAKE_CASE			:      Tuple 							=	fx_model_loaded(**__A						).to_tuple()
         self.assertEqual(
             len(__A						)						,							len(__A						)						,							'''Output lengths differ between Flax and PyTorch'''						)
         for fx_output_loaded, pt_output in zip(__A						,							__A						):
          self.assert_almost_equals(fx_output_loaded[:, -1]						,							pt_output[:, -1].numpy()						,							4e-2						)

      @is_pt_flax_cross_test
      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
        with self.subTest(model_class.__name__						):
         # prepare inputs
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	self._prepare_for_class(__A						,							__A						)
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	{k: torch.tensor(v.tolist()						) for k, v in prepared_inputs_dict.items()}

         # load corresponding PyTorch class
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	model_class.__name__[4:]  # Skip the "Flax" at the beginning
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	getattr(__A						,							__A						)

         __SCREAMING_SNAKE_CASE			:      List[str] 							=	pt_model_class(__A						).eval()
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	False
         __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	model_class(__A						,							dtype=jnp.floataa						)

         __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	load_flax_weights_in_pytorch_model(__A						,							fx_model.params						)
         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	pt_inputs["input_ids"].shape
         __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	np.random.randint(0						,							seq_length - 1						,							size=(batch_size,)						)
         for batch_idx, start_index in enumerate(__A						):
          __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	0
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	1
          __SCREAMING_SNAKE_CASE			:      Dict 							=	0
          __SCREAMING_SNAKE_CASE			:      Any 							=	1

         # make sure weights are tied in PyTorch
         pt_model.tie_weights()

         with torch.no_grad():
          __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	pt_model(**__A						).to_tuple()

         __SCREAMING_SNAKE_CASE			:      Tuple 							=	fx_model(**__A						).to_tuple()
         self.assertEqual(len(__A						)						,							len(__A						)						,							'''Output lengths differ between Flax and PyTorch'''						)
         for fx_output, pt_output in zip(__A						,							__A						):
          self.assert_almost_equals(fx_output[:, -1]						,							pt_output[:, -1].numpy()						,							4e-2						)

         with tempfile.TemporaryDirectory() as tmpdirname:
          fx_model.save_pretrained(__A						)
          __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	pt_model_class.from_pretrained(__A						,							from_flax=__A						)

         with torch.no_grad():
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	pt_model_loaded(**__A						).to_tuple()

         self.assertEqual(
             len(__A						)						,							len(__A						)						,							'''Output lengths differ between Flax and PyTorch'''						)
         for fx_output, pt_output in zip(__A						,							__A						):
          self.assert_almost_equals(fx_output[:, -1]						,							pt_output[:, -1].numpy()						,							4e-2						)



      @slow
      def 						SCREAMING_SNAKE_CASE_      (      self       :int						):





       for model_class_name in self.all_model_classes:
        __SCREAMING_SNAKE_CASE			:      str 							=	model_class_name.from_pretrained('''facebook/xglm-564M'''						)
        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	model(np.ones((1, 1)						)						)
        self.assertIsNotNone(__A						)








"""simple docstring"""





import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       snake_case		(       __UpperCAmelCase			,					unittest.TestCase       ):
      lowerCamelCase__      =			PreTrainedTokenizerFast
      lowerCamelCase__      =			False
      lowerCamelCase__      =			True
      lowerCamelCase__      =			'''tokenizer_file'''

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	False  # because we don't have pretrained_vocab_files_map
       super().setUp()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	True

       __SCREAMING_SNAKE_CASE			:      int 							=	['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

       # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

       __SCREAMING_SNAKE_CASE			:      str 							=	PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
       tokenizer.save_pretrained(self.tmpdirname						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
       # model
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
       pass

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      int 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	tmpdirname_orig

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.tmpdirname
       # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         try:
          __SCREAMING_SNAKE_CASE			:      Dict 							=	tempfile.mkdtemp()
          __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase						,							**_lowerCamelCase						)

          tokenizer.save_pretrained(self.tmpdirname						)
          super().test_training_new_tokenizer_with_special_tokens_change()
         finally:
          # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
          # is restored
          shutil.rmtree(self.tmpdirname						)
          __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tmpdirname_orig



      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						):
       __SCREAMING_SNAKE_CASE			:      str 							=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	('''a''' for _ in range(1_0_0_0						))
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase						,							length=1_0_0_0						,							vocab_size=5_0						)

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	new_tokenizer.encode('''a🤗'''						)
       self.assertEqual(_lowerCamelCase						,							[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained('''bert-base-cased'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	json.loads(tokenizer._tokenizer.to_str()						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	len(_lowerCamelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
        # Hack to save this in the tokenizer_config.json
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	['''tokenizer.4.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        json.dump(_lowerCamelCase						,							open(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							'''w'''						)						)

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        __SCREAMING_SNAKE_CASE			:      Dict 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						) + 1						)
        __SCREAMING_SNAKE_CASE			:      Dict 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
        shutil.move(os.path.join(_lowerCamelCase						,							'''tokenizer.4.0.0.json'''						)						,							os.path.join(_lowerCamelCase						,							'''tokenizer.42.0.0.json'''						)						)
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	['''tokenizer.42.0.0.json''']
        tokenizer.save_pretrained(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
        self.assertEqual(len(_lowerCamelCase						)						,							len(_lowerCamelCase						)						)
        __SCREAMING_SNAKE_CASE			:      List[str] 							=	json.loads(new_tokenizer._tokenizer.to_str()						)
        self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	'''hf-internal-testing/test-two-tokenizers'''

       # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
       __SCREAMING_SNAKE_CASE			:      int 							=	AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_7						)
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	json.loads(tokenizer._tokenizer.to_str()						)
       self.assertIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)

       # Testing an older version by monkey-patching the version in the module it's used.
       import transformers as old_transformers

       __SCREAMING_SNAKE_CASE			:      Dict 							=	'''3.0.0'''
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase						)
       self.assertEqual(len(_lowerCamelCase						)						,							2_8_9_9_6						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	json.loads(old_tokenizer._tokenizer.to_str()						)
       self.assertNotIn('''huggingface'''						,							json_tokenizer['''model''']['''vocab''']						)







@require_tokenizers
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       # See https://github.com/huggingface/transformers/pull/12550
       # and https://github.com/huggingface/tokenizers/issues/537
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''						)
       __SCREAMING_SNAKE_CASE			:      str 							=	'''The Matrix is a 1999 science fiction action film.'''

       with concurrent.futures.ThreadPoolExecutor() as executor:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	[executor.submit(self.fetch						,							_lowerCamelCase						,							_lowerCamelCase						) for i in range(1_0						)]
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	[future.result() for future in futures]
        self.assertEqual(_lowerCamelCase						,							[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0						)]						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Optional[int]						):
       return tokenizer.encode(_lowerCamelCase						,							truncation='''longest_first'''						,							padding='''longest'''						)

