






"""simple docstring"""




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    PreTrainedTokenizerFast
							_UpperCAmelCase	      =    False
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    "tokenizer_file"
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Optional[Any]:
											snake_case__													=       False  # because we don't have pretrained_vocab_files_map
											super().setUp()
											snake_case__													=       True

											snake_case__													=       ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
											snake_case__													=       'SaulLu/dummy-tokenizer-bytelevel-bpe'

											# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
											snake_case__													=       [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

											snake_case__													=       PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
											tokenizer.save_pretrained(self.tmpdirname  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Dict:
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass
							def lowerCAmelCase_	(       self: str  )						->						Union[str, Any]:
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass
							def lowerCAmelCase_	(       self: Any  )						->						Tuple:
											# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
											# model
											pass
							def lowerCAmelCase_	(       self: int  )						->						Optional[Any]:
											# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
											pass
							def lowerCAmelCase_	(       self: Tuple  )						->						Optional[int]:
											snake_case__													=       self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			try:
																							snake_case__													=       tempfile.mkdtemp()
																							snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																							tokenizer.save_pretrained(self.tmpdirname  )
																							super().test_training_new_tokenizer()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname  )
																							snake_case__													=       tmpdirname_orig
							def lowerCAmelCase_	(       self: Union[str, Any]  )						->						Optional[int]:
											snake_case__													=       self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			try:
																							snake_case__													=       tempfile.mkdtemp()
																							snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																							tokenizer.save_pretrained(self.tmpdirname  )
																							super().test_training_new_tokenizer_with_special_tokens_change()
																			finally:
																							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																							# is restored
																							shutil.rmtree(self.tmpdirname  )
																							snake_case__													=       tmpdirname_orig


							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Union[str, Any]:
											snake_case__													=       self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

											snake_case__													=       ('a' for _ in range(10_00  ))
											snake_case__													=       tokenizer.train_new_from_iterator(text_iterator=UpperCamelCase    ,		length=10_00    ,		vocab_size=50  )

											snake_case__													=       new_tokenizer.encode('a🤗'  )
											self.assertEqual(UpperCamelCase    ,		[64, 1_72, 2_53, 97, 2_45]  )


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def lowerCAmelCase_	(       self: List[Any]  )						->						Optional[int]:
											snake_case__													=       AutoTokenizer.from_pretrained('bert-base-cased'  )
											snake_case__													=       json.loads(tokenizer._tokenizer.to_str()  )
											snake_case__													=       len(UpperCamelCase  )

											with tempfile.TemporaryDirectory() as tmp_dir:
															# Hack to save this in the tokenizer_config.json
															snake_case__													=       ['tokenizer.4.0.0.json']
															tokenizer.save_pretrained(UpperCamelCase  )
															json.dump(UpperCamelCase    ,		open(os.path.join(UpperCamelCase    ,		'tokenizer.4.0.0.json'  )    ,		'w'  )  )

															# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
															snake_case__													=       AutoTokenizer.from_pretrained(UpperCamelCase  )
															self.assertEqual(len(UpperCamelCase  )    ,		len(UpperCamelCase  ) + 1  )
															snake_case__													=       json.loads(new_tokenizer._tokenizer.to_str()  )
															self.assertIn('huggingface'    ,		json_tokenizer['model']['vocab']  )

															# Will need to be adjusted if we reach v42 and this test is still here.
															# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
															shutil.move(os.path.join(UpperCamelCase    ,		'tokenizer.4.0.0.json'  )    ,		os.path.join(UpperCamelCase    ,		'tokenizer.42.0.0.json'  )  )
															snake_case__													=       ['tokenizer.42.0.0.json']
															tokenizer.save_pretrained(UpperCamelCase  )
															snake_case__													=       AutoTokenizer.from_pretrained(UpperCamelCase  )
															self.assertEqual(len(UpperCamelCase  )    ,		len(UpperCamelCase  )  )
															snake_case__													=       json.loads(new_tokenizer._tokenizer.to_str()  )
															self.assertNotIn('huggingface'    ,		json_tokenizer['model']['vocab']  )


							def lowerCAmelCase_	(       self: Any  )						->						Tuple:
											# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
											snake_case__													=       'hf-internal-testing/test-two-tokenizers'

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											snake_case__													=       AutoTokenizer.from_pretrained(UpperCamelCase  )
											self.assertEqual(len(UpperCamelCase  )    ,		2_89_97  )
											snake_case__													=       json.loads(tokenizer._tokenizer.to_str()  )
											self.assertIn('huggingface'    ,		json_tokenizer['model']['vocab']  )

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											snake_case__													=       '3.0.0'
											snake_case__													=       old_transformers.models.auto.AutoTokenizer.from_pretrained(UpperCamelCase  )
											self.assertEqual(len(UpperCamelCase  )    ,		2_89_96  )
											snake_case__													=       json.loads(old_tokenizer._tokenizer.to_str()  )
											self.assertNotIn('huggingface'    ,		json_tokenizer['model']['vocab']  )


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def lowerCAmelCase_	(       self: Optional[Any]  )						->						Any:
											# See https://github.com/huggingface/transformers/pull/12550
											# and https://github.com/huggingface/tokenizers/issues/537
											snake_case__													=       PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'  )
											snake_case__													=       'The Matrix is a 1999 science fiction action film.'

											with concurrent.futures.ThreadPoolExecutor() as executor:
															snake_case__													=       [executor.submit(self.fetch    ,		UpperCamelCase    ,		UpperCamelCase  ) for i in range(10  )]
															snake_case__													=       [future.result() for future in futures]
															self.assertEqual(UpperCamelCase    ,		[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )


							def lowerCAmelCase_	(       self: List[Any]    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: List[Any]  )						->						str:
											return tokenizer.encode(UpperCamelCase    ,		truncation='longest_first'    ,		padding='longest'  )












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCamelCase   :						int							=  {
    """configuration_openai""": ["""OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """OpenAIGPTConfig"""],
    """tokenization_openai""": ["""OpenAIGPTTokenizer"""],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase   :						Optional[Any]							=  ["""OpenAIGPTTokenizerFast"""]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase   :						List[str]							=  [
							    """OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """OpenAIGPTDoubleHeadsModel""",
							    """OpenAIGPTForSequenceClassification""",
							    """OpenAIGPTLMHeadModel""",
							    """OpenAIGPTModel""",
							    """OpenAIGPTPreTrainedModel""",
							    """load_tf_weights_in_openai_gpt""",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase   :						Dict							=  [
							    """TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFOpenAIGPTDoubleHeadsModel""",
							    """TFOpenAIGPTForSequenceClassification""",
							    """TFOpenAIGPTLMHeadModel""",
							    """TFOpenAIGPTMainLayer""",
							    """TFOpenAIGPTModel""",
							    """TFOpenAIGPTPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
							from .tokenization_openai import OpenAIGPTTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_openai_fast import OpenAIGPTTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_openai import (
														    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    OpenAIGPTDoubleHeadsModel,
														    OpenAIGPTForSequenceClassification,
														    OpenAIGPTLMHeadModel,
														    OpenAIGPTModel,
														    OpenAIGPTPreTrainedModel,
														    load_tf_weights_in_openai_gpt,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_openai import (
														    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFOpenAIGPTDoubleHeadsModel,
														    TFOpenAIGPTForSequenceClassification,
														    TFOpenAIGPTLMHeadModel,
														    TFOpenAIGPTMainLayer,
														    TFOpenAIGPTModel,
														    TFOpenAIGPTPreTrainedModel,
														)

else:
							import sys

							__UpperCamelCase   :						List[Any]							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




