





"""simple docstring"""




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 			_UpperCAmelCase  (       A__      ,       unittest.TestCase		):



      '''simple docstring'''
      lowercase_				:			str        =       PreTrainedTokenizerFast
      lowercase_				:			Optional[Any]        =       False
      lowercase_				:			Any        =       True
      lowercase_				:			int        =       """tokenizer_file"""

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	List[str]    						=  False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        A_  :	List[Any]    						=  True

        A_  :	str    						=  ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
        A_  :	Dict    						=  '''SaulLu/dummy-tokenizer-bytelevel-bpe'''

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        A_  :	Optional[Any]    						=  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        A_  :	Optional[Any]    						=  PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
        tokenizer.save_pretrained(self.tmpdirname  )

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        pass

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        pass

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        pass

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        pass

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	List[Any]    						=  self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
            try:
              A_  :	Dict    						=  tempfile.mkdtemp()
              A_  :	List[str]    						=  self.rust_tokenizer_class.from_pretrained(__snake_case ,				**__snake_case  )

              tokenizer.save_pretrained(self.tmpdirname  )
              super().test_training_new_tokenizer()
            finally:
              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
              # is restored
              shutil.rmtree(self.tmpdirname  )
              A_  :	int    						=  tmpdirname_orig

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	str    						=  self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
            try:
              A_  :	List[Any]    						=  tempfile.mkdtemp()
              A_  :	Union[str, Any]    						=  self.rust_tokenizer_class.from_pretrained(__snake_case ,				**__snake_case  )

              tokenizer.save_pretrained(self.tmpdirname  )
              super().test_training_new_tokenizer_with_special_tokens_change()
            finally:
              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
              # is restored
              shutil.rmtree(self.tmpdirname  )
              A_  :	Union[str, Any]    						=  tmpdirname_orig

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	str    						=  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

        A_  :	str    						=  ('''a''' for _ in range(1_0_0_0  ))
        A_  :	Optional[int]    						=  tokenizer.train_new_from_iterator(text_iterator=__snake_case ,				length=1_0_0_0 ,				vocab_size=5_0  )

        A_  :	Union[str, Any]    						=  new_tokenizer.encode('a🤗'  )
        self.assertEqual(__snake_case ,				[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]  )


@require_tokenizers
class 			_UpperCAmelCase  (       unittest.TestCase		):



      '''simple docstring'''

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	Any    						=  AutoTokenizer.from_pretrained('bert-base-cased'  )
        A_  :	int    						=  json.loads(tokenizer._tokenizer.to_str()  )
        A_  :	List[str]    						=  len(__snake_case  )

        with tempfile.TemporaryDirectory() as tmp_dir:
          # Hack to save this in the tokenizer_config.json
          A_  :	Any    						=  ['''tokenizer.4.0.0.json''']
          tokenizer.save_pretrained(__snake_case  )
          json.dump(__snake_case ,				open(os.path.join(__snake_case ,				'tokenizer.4.0.0.json'  ) ,				'w'  )  )

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          A_  :	Optional[int]    						=  AutoTokenizer.from_pretrained(__snake_case  )
          self.assertEqual(len(__snake_case  ) ,				len(__snake_case  ) + 1  )
          A_  :	Any    						=  json.loads(new_tokenizer._tokenizer.to_str()  )
          self.assertIn('huggingface' ,				json_tokenizer['model']['vocab']  )

          # Will need to be adjusted if we reach v42 and this test is still here.
          # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
          shutil.move(os.path.join(__snake_case ,				'tokenizer.4.0.0.json'  ) ,				os.path.join(__snake_case ,				'tokenizer.42.0.0.json'  )  )
          A_  :	int    						=  ['''tokenizer.42.0.0.json''']
          tokenizer.save_pretrained(__snake_case  )
          A_  :	Union[str, Any]    						=  AutoTokenizer.from_pretrained(__snake_case  )
          self.assertEqual(len(__snake_case  ) ,				len(__snake_case  )  )
          A_  :	Optional[Any]    						=  json.loads(new_tokenizer._tokenizer.to_str()  )
          self.assertNotIn('huggingface' ,				json_tokenizer['model']['vocab']  )

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	Dict    						=  '''hf-internal-testing/test-two-tokenizers'''

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        A_  :	Any    						=  AutoTokenizer.from_pretrained(__snake_case  )
        self.assertEqual(len(__snake_case  ) ,				2_8_9_9_7  )
        A_  :	Any    						=  json.loads(tokenizer._tokenizer.to_str()  )
        self.assertIn('huggingface' ,				json_tokenizer['model']['vocab']  )

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        A_  :	str    						=  '''3.0.0'''
        A_  :	List[Any]    						=  old_transformers.models.auto.AutoTokenizer.from_pretrained(__snake_case  )
        self.assertEqual(len(__snake_case  ) ,				2_8_9_9_6  )
        A_  :	Optional[int]    						=  json.loads(old_tokenizer._tokenizer.to_str()  )
        self.assertNotIn('huggingface' ,				json_tokenizer['model']['vocab']  )


@require_tokenizers
class 			_UpperCAmelCase  (       unittest.TestCase		):



      '''simple docstring'''

      def  lowerCamelCase_      (					self  ):

        """simple docstring"""

        A_  :	Tuple    						=  PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'  )
        A_  :	Optional[int]    						=  '''The Matrix is a 1999 science fiction action film.'''

        with concurrent.futures.ThreadPoolExecutor() as executor:
          A_  :	List[Any]    						=  [executor.submit(self.fetch ,				__snake_case ,				__snake_case  ) for i in range(1_0  )]
          A_  :	str    						=  [future.result() for future in futures]
          self.assertEqual(__snake_case ,				[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0  )]  )

      def  lowerCamelCase_      (					self ,				snake_case_ ,				snake_case_  ):

        """simple docstring"""

        return tokenizer.encode(__snake_case ,				truncation='longest_first' ,				padding='longest'  )






"""simple docstring"""




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase_					:						Any      = {
    'configuration_tvlt': ['TVLT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TvltConfig'],
    'feature_extraction_tvlt': ['TvltFeatureExtractor'],
    'processing_tvlt': ['TvltProcessor'],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase_					:						Union[str, Any]      = [
       'TVLT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'TvltModel',
       'TvltForPreTraining',
       'TvltForAudioVisualClassification',
       'TvltPreTrainedModel',
   ]

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase_					:						List[Any]      = ['TvltImageProcessor']


if TYPE_CHECKING:
   from .configuration_tvlt import TVLT_PRETRAINED_CONFIG_ARCHIVE_MAP, TvltConfig
   from .processing_tvlt import TvltProcessor
   from .feature_extraction_tvlt import TvltFeatureExtractor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tvlt import (
          TVLT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TvltForAudioVisualClassification,
          TvltForPreTraining,
          TvltModel,
          TvltPreTrainedModel,
      )

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_tvlt import TvltImageProcessor


else:
   import sys

   lowerCamelCase_					:						Optional[int]      = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)