





'''simple docstring'''


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 				lowerCAmelCase_						(						__magic_name__  ,unittest.TestCase						):
	__lowerCamelCase  :							Dict    					=   PreTrainedTokenizerFast
	__lowerCamelCase  :							List[Any]    					=   False
	__lowerCamelCase  :							List[str]    					=   True
	__lowerCamelCase  :							Union[str, Any]    					=   "tokenizer_file"




	def 							_snake_case				(		self						)      ->							Optional[Any]:
					_lowerCAmelCase		     = False  # because we don't have pretrained_vocab_files_map
					super().setUp()
					_lowerCAmelCase		     = True

					_lowerCAmelCase		     = ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
					_lowerCAmelCase		     = "SaulLu/dummy-tokenizer-bytelevel-bpe"

					# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
					_lowerCAmelCase		     = [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

					_lowerCAmelCase		     = PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
					tokenizer.save_pretrained(self.tmpdirname						)




	def 							_snake_case				(		self						)      ->							Tuple:
					# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
					# model
					pass




	def 							_snake_case				(		self						)      ->							Any:
					# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
					# model
					pass




	def 							_snake_case				(		self						)      ->							int:
					# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
					# model
					pass




	def 							_snake_case				(		self						)      ->							Optional[Any]:
					# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
					pass




	def 							_snake_case				(		self						)      ->							Union[str, Any]:
					_lowerCAmelCase		     = self.tmpdirname
					# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													try:
																	_lowerCAmelCase		     = tempfile.mkdtemp()
																	_lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

																	tokenizer.save_pretrained(self.tmpdirname						)
																	super().test_training_new_tokenizer()
													finally:
																	# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																	# is restored
																	shutil.rmtree(self.tmpdirname						)
																	_lowerCAmelCase		     = tmpdirname_orig




	def 							_snake_case				(		self						)      ->							Optional[int]:
					_lowerCAmelCase		     = self.tmpdirname
					# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													try:
																	_lowerCAmelCase		     = tempfile.mkdtemp()
																	_lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase     ,       **_lowerCAmelCase						)

																	tokenizer.save_pretrained(self.tmpdirname						)
																	super().test_training_new_tokenizer_with_special_tokens_change()
													finally:
																	# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																	# is restored
																	shutil.rmtree(self.tmpdirname						)
																	_lowerCAmelCase		     = tmpdirname_orig





	def 							_snake_case				(		self						)      ->							Union[str, Any]:
					_lowerCAmelCase		     = self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

					_lowerCAmelCase		     = ("a" for _ in range(1000						))
					_lowerCAmelCase		     = tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase     ,       length=1000     ,       vocab_size=50						)

					_lowerCAmelCase		     = new_tokenizer.encode("a🤗"						)
					self.assertEqual(_lowerCAmelCase     ,       [64, 172, 253, 97, 245]						)







@require_tokenizers
class 				lowerCAmelCase_						(						unittest.TestCase						):




	def 							_snake_case				(		self						)      ->							Tuple:
					_lowerCAmelCase		     = AutoTokenizer.from_pretrained("bert-base-cased"						)
					_lowerCAmelCase		     = json.loads(tokenizer._tokenizer.to_str()						)
					_lowerCAmelCase		     = len(_lowerCAmelCase						)

					with tempfile.TemporaryDirectory() as tmp_dir:
									# Hack to save this in the tokenizer_config.json
									_lowerCAmelCase		     = ["tokenizer.4.0.0.json"]
									tokenizer.save_pretrained(_lowerCAmelCase						)
									json.dump(_lowerCAmelCase     ,       open(os.path.join(_lowerCAmelCase     ,       "tokenizer.4.0.0.json"						)     ,       "w"						)						)

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									_lowerCAmelCase		     = AutoTokenizer.from_pretrained(_lowerCAmelCase						)
									self.assertEqual(len(_lowerCAmelCase						)     ,       len(_lowerCAmelCase						) + 1						)
									_lowerCAmelCase		     = json.loads(new_tokenizer._tokenizer.to_str()						)
									self.assertIn("huggingface"     ,       json_tokenizer["model"]["vocab"]						)

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
									shutil.move(os.path.join(_lowerCAmelCase     ,       "tokenizer.4.0.0.json"						)     ,       os.path.join(_lowerCAmelCase     ,       "tokenizer.42.0.0.json"						)						)
									_lowerCAmelCase		     = ["tokenizer.42.0.0.json"]
									tokenizer.save_pretrained(_lowerCAmelCase						)
									_lowerCAmelCase		     = AutoTokenizer.from_pretrained(_lowerCAmelCase						)
									self.assertEqual(len(_lowerCAmelCase						)     ,       len(_lowerCAmelCase						)						)
									_lowerCAmelCase		     = json.loads(new_tokenizer._tokenizer.to_str()						)
									self.assertNotIn("huggingface"     ,       json_tokenizer["model"]["vocab"]						)





	def 							_snake_case				(		self						)      ->							Union[str, Any]:
					# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
					_lowerCAmelCase		     = "hf-internal-testing/test-two-tokenizers"

					# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
					_lowerCAmelCase		     = AutoTokenizer.from_pretrained(_lowerCAmelCase						)
					self.assertEqual(len(_lowerCAmelCase						)     ,       28997						)
					_lowerCAmelCase		     = json.loads(tokenizer._tokenizer.to_str()						)
					self.assertIn("huggingface"     ,       json_tokenizer["model"]["vocab"]						)

					# Testing an older version by monkey-patching the version in the module it's used.
					import transformers as old_transformers

					_lowerCAmelCase		     = "3.0.0"
					_lowerCAmelCase		     = old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase						)
					self.assertEqual(len(_lowerCAmelCase						)     ,       28996						)
					_lowerCAmelCase		     = json.loads(old_tokenizer._tokenizer.to_str()						)
					self.assertNotIn("huggingface"     ,       json_tokenizer["model"]["vocab"]						)







@require_tokenizers
class 				lowerCAmelCase_						(						unittest.TestCase						):




	def 							_snake_case				(		self						)      ->							Dict:
					# See https://github.com/huggingface/transformers/pull/12550
					# and https://github.com/huggingface/tokenizers/issues/537
					_lowerCAmelCase		     = PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"						)
					_lowerCAmelCase		     = "The Matrix is a 1999 science fiction action film."

					with concurrent.futures.ThreadPoolExecutor() as executor:
									_lowerCAmelCase		     = [executor.submit(self.fetch     ,       _lowerCAmelCase     ,       _lowerCAmelCase						) for i in range(10						)]
									_lowerCAmelCase		     = [future.result() for future in futures]
									self.assertEqual(_lowerCAmelCase     ,       [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10						)]						)





	def 							_snake_case				(		self     ,       _lowerCAmelCase     ,       _lowerCAmelCase						)      ->							Tuple:
					return tokenizer.encode(_lowerCAmelCase     ,       truncation="longest_first"     ,       padding="longest"						)









'''simple docstring'''


from __future__ import annotations

from collections.abc import Iterable, Iterator
from typing import Any, Generic, TypeVar

_SCREAMING_SNAKE_CASE			=							TypeVar("T", bound=bool)

class 				lowerCAmelCase_						(						Generic[T]						):




 def __init__(		self     ,       _lowerCAmelCase						)      ->							None:
     _lowerCAmelCase		     = value
     _lowerCAmelCase		     = None
     _lowerCAmelCase		     = None




 @property
 def 							_snake_case				(		self						)      ->							T:
     return self._value





 @staticmethod
 def 							_snake_case				(		_lowerCAmelCase     ,       _lowerCAmelCase						)      ->							SkewNode[T] | None:
     if not roota:
         return roota

     if not roota:
         return roota

     if roota.value > roota.value:
         _lowerCAmelCase     ,     _lowerCAmelCase		     = roota, roota

     _lowerCAmelCase		     = roota
     _lowerCAmelCase		     = roota.right
     _lowerCAmelCase		     = roota.left
     _lowerCAmelCase		     = SkewNode.merge(_lowerCAmelCase     ,       _lowerCAmelCase						)

     return result







class 				lowerCAmelCase_						(						Generic[T]						):




 def __init__(		self     ,       _lowerCAmelCase = ()						)      ->							None:
     _lowerCAmelCase		     = None
     if data:
         for item in data:
             self.insert(_lowerCAmelCase						)




 def __bool__(		self						)      ->							bool:
     return self._root is not None




 def __iter__(		self						)      ->							Iterator[T]:
     _lowerCAmelCase		     = []
     while self:
         result.append(self.pop()						)

     # Pushing items back to the heap not to clear it.
     for item in result:
         self.insert(_lowerCAmelCase						)

     return iter(_lowerCAmelCase						)




 def 							_snake_case				(		self     ,       _lowerCAmelCase						)      ->							None:
     _lowerCAmelCase		     = SkewNode.merge(self._root     ,       SkewNode(_lowerCAmelCase						)						)




 def 							_snake_case				(		self						)      ->							T | None:
     _lowerCAmelCase		     = self.top()
     _lowerCAmelCase		     = (
         SkewNode.merge(self._root.left     ,       self._root.right						) if self._root else None
     )

     return result




 def 							_snake_case				(		self						)      ->							T:
     if not self._root:
         raise IndexError("Can't get top element for the empty heap."						)
     return self._root.value





 def 							_snake_case				(		self						)      ->							None:
     _lowerCAmelCase		     = None


if __name__ == "__main__":
       import doctest

       doctest.testmod()


