




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE							,     unittest.TestCase				):

      """simple docstring"""

      lowerCAmelCase__     :	Optional[Any]      =   PreTrainedTokenizerFast
      lowerCAmelCase__     :	Optional[Any]      =   False
      lowerCAmelCase__     :	int      =   True
      lowerCAmelCase__     :	Union[str, Any]      =   "tokenizer_file"







      def  snake_case		(       self   :	Dict			):
          __UpperCamelCase											=  False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          __UpperCamelCase											=  True

          __UpperCamelCase											=  ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
          __UpperCamelCase											=  '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          __UpperCamelCase											=  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          __UpperCamelCase											=  PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
          tokenizer.save_pretrained(self.tmpdirname			)







      def  snake_case		(       self   :	Optional[int]			):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass







      def  snake_case		(       self   :	Dict			):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass







      def  snake_case		(       self   :	Optional[int]			):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass







      def  snake_case		(       self   :	int			):
          # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
          pass







      def  snake_case		(       self   :	List[Any]			):
          __UpperCamelCase											=  self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"			):
                  try:
                      __UpperCamelCase											=  tempfile.mkdtemp()
                      __UpperCamelCase											=  self.rust_tokenizer_class.from_pretrained(snake_case	,    **snake_case			)

                      tokenizer.save_pretrained(self.tmpdirname			)
                      super().test_training_new_tokenizer()
                  finally:
                      # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                      # is restored
                      shutil.rmtree(self.tmpdirname			)
                      __UpperCamelCase											=  tmpdirname_orig







      def  snake_case		(       self   :	List[Any]			):
          __UpperCamelCase											=  self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"			):
                  try:
                      __UpperCamelCase											=  tempfile.mkdtemp()
                      __UpperCamelCase											=  self.rust_tokenizer_class.from_pretrained(snake_case	,    **snake_case			)

                      tokenizer.save_pretrained(self.tmpdirname			)
                      super().test_training_new_tokenizer_with_special_tokens_change()
                  finally:
                      # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                      # is restored
                      shutil.rmtree(self.tmpdirname			)
                      __UpperCamelCase											=  tmpdirname_orig







      def  snake_case		(       self   :	str			):
          __UpperCamelCase											=  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

          __UpperCamelCase											=  ('''a''' for _ in range(1000			))
          __UpperCamelCase											=  tokenizer.train_new_from_iterator(text_iterator=snake_case	,    length=1000	,    vocab_size=50			)

          __UpperCamelCase											=  new_tokenizer.encode('''a🤗'''			)
          self.assertEqual(snake_case	,    [64, 172, 253, 97, 245]			)







@require_tokenizers
class 	_lowerCamelCase  (     unittest.TestCase				):

      """simple docstring"""

      def  snake_case		(       self   :	int			):
          __UpperCamelCase											=  AutoTokenizer.from_pretrained('''bert-base-cased'''			)
          __UpperCamelCase											=  json.loads(tokenizer._tokenizer.to_str()			)
          __UpperCamelCase											=  len(snake_case			)

          with tempfile.TemporaryDirectory() as tmp_dir:
              # Hack to save this in the tokenizer_config.json
              __UpperCamelCase											=  ['''tokenizer.4.0.0.json''']
              tokenizer.save_pretrained(snake_case			)
              json.dump(snake_case	,    open(os.path.join(snake_case	,    '''tokenizer.4.0.0.json'''			)	,    '''w'''			)			)

              # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
              __UpperCamelCase											=  AutoTokenizer.from_pretrained(snake_case			)
              self.assertEqual(len(snake_case			)	,    len(snake_case			) + 1			)
              __UpperCamelCase											=  json.loads(new_tokenizer._tokenizer.to_str()			)
              self.assertIn('''huggingface'''	,    json_tokenizer['''model''']['''vocab''']			)

              # Will need to be adjusted if we reach v42 and this test is still here.
              # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
              shutil.move(os.path.join(snake_case	,    '''tokenizer.4.0.0.json'''			)	,    os.path.join(snake_case	,    '''tokenizer.42.0.0.json'''			)			)
              __UpperCamelCase											=  ['''tokenizer.42.0.0.json''']
              tokenizer.save_pretrained(snake_case			)
              __UpperCamelCase											=  AutoTokenizer.from_pretrained(snake_case			)
              self.assertEqual(len(snake_case			)	,    len(snake_case			)			)
              __UpperCamelCase											=  json.loads(new_tokenizer._tokenizer.to_str()			)
              self.assertNotIn('''huggingface'''	,    json_tokenizer['''model''']['''vocab''']			)







      def  snake_case		(       self   :	Optional[Any]			):
          # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
          __UpperCamelCase											=  '''hf-internal-testing/test-two-tokenizers'''

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          __UpperCamelCase											=  AutoTokenizer.from_pretrained(snake_case			)
          self.assertEqual(len(snake_case			)	,    28997			)
          __UpperCamelCase											=  json.loads(tokenizer._tokenizer.to_str()			)
          self.assertIn('''huggingface'''	,    json_tokenizer['''model''']['''vocab''']			)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          __UpperCamelCase											=  '''3.0.0'''
          __UpperCamelCase											=  old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case			)
          self.assertEqual(len(snake_case			)	,    28996			)
          __UpperCamelCase											=  json.loads(old_tokenizer._tokenizer.to_str()			)
          self.assertNotIn('''huggingface'''	,    json_tokenizer['''model''']['''vocab''']			)







@require_tokenizers
class 	_lowerCamelCase  (     unittest.TestCase				):

      """simple docstring"""

      def  snake_case		(       self   :	int			):
          # See https://github.com/huggingface/transformers/pull/12550
          # and https://github.com/huggingface/tokenizers/issues/537
          __UpperCamelCase											=  PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''			)
          __UpperCamelCase											=  '''The Matrix is a 1999 science fiction action film.'''

          with concurrent.futures.ThreadPoolExecutor() as executor:
              __UpperCamelCase											=  [executor.submit(self.fetch	,    snake_case	,    snake_case			) for i in range(10			)]
              __UpperCamelCase											=  [future.result() for future in futures]
              self.assertEqual(snake_case	,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)







      def  snake_case		(       self   :	int	,    snake_case   :	str	,    snake_case   :	Optional[Any]			):
          return tokenizer.encode(snake_case	,    truncation='''longest_first'''	,    padding='''longest'''			)












def    __SCREAMING_SNAKE_CASE				(					lowercase_ )    ->		int:

    """simple docstring"""


    if not isinstance(lowercase_   ,       lowercase_ ):
        raise TypeError('''recursive_lucas_number accepts only integer arguments.''' )
    if n_th_number == 0:
        return 2
    if n_th_number == 1:
        return 1

    return recursive_lucas_number(n_th_number - 1 ) + recursive_lucas_number(
        n_th_number - 2 )




def    __SCREAMING_SNAKE_CASE				(					lowercase_ )    ->		int:

    """simple docstring"""


    if not isinstance(lowercase_   ,       lowercase_ ):
        raise TypeError('''dynamic_lucas_number accepts only integer arguments.''' )
    __UpperCamelCase						, __UpperCamelCase											=  2, 1
    for _ in range(lowercase_ ):
        __UpperCamelCase						, __UpperCamelCase											=  b, a + b
    return a


if __name__ == "__main__":
 from doctest import testmod

 testmod()
 a_										=       int(input("Enter the number of terms in lucas series:\n").strip())
 print("Using recursive function to calculate lucas series:")
 print(" ".join(str(recursive_lucas_number(i)) for i in range(n)))
 print("\nUsing dynamic function to calculate lucas series:")
 print(" ".join(str(dynamic_lucas_number(i)) for i in range(n)))






