import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class        _UpperCamelCase				(						_UpperCAmelCase						,unittest.TestCase			):


			"""simple docstring"""






			__a  :			Any      			=						PreTrainedTokenizerFast
			__a  :			List[Any]      			=						False
			__a  :			Any      			=						True
			__a  :			int      			=						'''tokenizer_file'''





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> Dict:

							'''simple docstring'''


							__lowercase			      =							False  # because we don't have pretrained_vocab_files_map
							super().setUp()
							__lowercase			      =							True

							__lowercase			      =							['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
							__lowercase			      =							'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

							# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
							__lowercase			      =							[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

							__lowercase			      =							PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
							tokenizer.save_pretrained(self.tmpdirname						)





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> List[str]:

							'''simple docstring'''


							pass





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> str:

							'''simple docstring'''


							pass





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> Tuple:

							'''simple docstring'''


							pass





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> str:

							'''simple docstring'''


							pass





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> int:

							'''simple docstring'''


							__lowercase			      =							self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
															try:
																			__lowercase			      =							tempfile.mkdtemp()
																			__lowercase			      =							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__				,       **lowerCAmelCase__						)

																			tokenizer.save_pretrained(self.tmpdirname						)
																			super().test_training_new_tokenizer()
															finally:
																			# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																			# is restored
																			shutil.rmtree(self.tmpdirname						)
																			__lowercase			      =							tmpdirname_orig





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> Any:

							'''simple docstring'''


							__lowercase			      =							self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
															try:
																			__lowercase			      =							tempfile.mkdtemp()
																			__lowercase			      =							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__				,       **lowerCAmelCase__						)

																			tokenizer.save_pretrained(self.tmpdirname						)
																			super().test_training_new_tokenizer_with_special_tokens_change()
															finally:
																			# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																			# is restored
																			shutil.rmtree(self.tmpdirname						)
																			__lowercase			      =							tmpdirname_orig





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> List[Any]:

							'''simple docstring'''


							__lowercase			      =							self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

							__lowercase			      =							('''a''' for _ in range(10_00						))
							__lowercase			      =							tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__				,       length=10_00				,       vocab_size=50						)

							__lowercase			      =							new_tokenizer.encode('''a🤗'''						)
							self.assertEqual(lowerCAmelCase__				,       [64, 1_72, 2_53, 97, 2_45]						)




@require_tokenizers
class        _UpperCamelCase				(						unittest.TestCase			):


			"""simple docstring"""






			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> Any:

							'''simple docstring'''


							__lowercase			      =							AutoTokenizer.from_pretrained('''bert-base-cased'''						)
							__lowercase			      =							json.loads(tokenizer._tokenizer.to_str()						)
							__lowercase			      =							len(lowerCAmelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
											# Hack to save this in the tokenizer_config.json
											__lowercase			      =							['''tokenizer.4.0.0.json''']
											tokenizer.save_pretrained(lowerCAmelCase__						)
											json.dump(lowerCAmelCase__				,       open(os.path.join(lowerCAmelCase__				,       '''tokenizer.4.0.0.json'''						)				,       '''w'''						)						)

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											__lowercase			      =							AutoTokenizer.from_pretrained(lowerCAmelCase__						)
											self.assertEqual(len(lowerCAmelCase__						)				,       len(lowerCAmelCase__						) + 1						)
											__lowercase			      =							json.loads(new_tokenizer._tokenizer.to_str()						)
											self.assertIn('''huggingface'''				,       json_tokenizer['''model''']['''vocab''']						)

											# Will need to be adjusted if we reach v42 and this test is still here.
											# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
											shutil.move(os.path.join(lowerCAmelCase__				,       '''tokenizer.4.0.0.json'''						)				,       os.path.join(lowerCAmelCase__				,       '''tokenizer.42.0.0.json'''						)						)
											__lowercase			      =							['''tokenizer.42.0.0.json''']
											tokenizer.save_pretrained(lowerCAmelCase__						)
											__lowercase			      =							AutoTokenizer.from_pretrained(lowerCAmelCase__						)
											self.assertEqual(len(lowerCAmelCase__						)				,       len(lowerCAmelCase__						)						)
											__lowercase			      =							json.loads(new_tokenizer._tokenizer.to_str()						)
											self.assertNotIn('''huggingface'''				,       json_tokenizer['''model''']['''vocab''']						)





			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> int:

							'''simple docstring'''


							__lowercase			      =							'''hf-internal-testing/test-two-tokenizers'''

							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
							__lowercase			      =							AutoTokenizer.from_pretrained(lowerCAmelCase__						)
							self.assertEqual(len(lowerCAmelCase__						)				,       2_89_97						)
							__lowercase			      =							json.loads(tokenizer._tokenizer.to_str()						)
							self.assertIn('''huggingface'''				,       json_tokenizer['''model''']['''vocab''']						)

							# Testing an older version by monkey-patching the version in the module it's used.
							import transformers as old_transformers

							__lowercase			      =							'''3.0.0'''
							__lowercase			      =							old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__						)
							self.assertEqual(len(lowerCAmelCase__						)				,       2_89_96						)
							__lowercase			      =							json.loads(old_tokenizer._tokenizer.to_str()						)
							self.assertNotIn('''huggingface'''				,       json_tokenizer['''model''']['''vocab''']						)




@require_tokenizers
class        _UpperCamelCase				(						unittest.TestCase			):


			"""simple docstring"""






			def 	_SCREAMING_SNAKE_CASE      (       self						)					-> List[str]:

							'''simple docstring'''


							__lowercase			      =							PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''						)
							__lowercase			      =							'''The Matrix is a 1999 science fiction action film.'''

							with concurrent.futures.ThreadPoolExecutor() as executor:
											__lowercase			      =							[executor.submit(self.fetch				,       lowerCAmelCase__				,       lowerCAmelCase__						) for i in range(10						)]
											__lowercase			      =							[future.result() for future in futures]
											self.assertEqual(lowerCAmelCase__				,       [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10						)]						)





			def 	_SCREAMING_SNAKE_CASE      (       self				,       lowerCAmelCase__				,       lowerCAmelCase__						)					-> int:

							'''simple docstring'''


							return tokenizer.encode(lowerCAmelCase__				,       truncation='''longest_first'''				,       padding='''longest'''						)
import timeit




class        _UpperCamelCase				:


			"""simple docstring"""






			def __init__(       self				,       lowerCAmelCase__						)					-> List[str]:

							'''simple docstring'''


							if isinstance(lowerCAmelCase__				,       lowerCAmelCase__						):  # Initializes a matrix identical to the one provided.
											__lowercase			      =							arg
											__lowercase			      =							len(lowerCAmelCase__						)
							else:  # Initializes a square matrix of the given size and set values to zero.
											__lowercase			      =							arg
											__lowercase			      =							[[0 for _ in range(self.n						)] for _ in range(self.n						)]





			def __mul__(       self				,       lowerCAmelCase__						)					-> str:

							'''simple docstring'''


							__lowercase			      =							Matrix(self.n						)
							for i in range(self.n						):
											for j in range(self.n						):
															for k in range(self.n						):
																			matrix.t[i][j] += self.t[i][k] * b.t[k][j]
							return matrix





def     UpperCAmelCase						(   lowercase							,      lowercase							):







				"""simple docstring"""



				__lowercase			      =							Matrix([[1, 0], [0, 1]]							)
				while b > 0:
								if b & 1:
												matrix *= a
								a *= a
								b >>= 1
				return matrix





def     UpperCAmelCase						(   lowercase							,      lowercase							,      lowercase							):







				"""simple docstring"""



				if n == 1:
								return fa
				elif n == 2:
								return fa
				__lowercase			      =							Matrix([[1, 1], [1, 0]]							)
				__lowercase			      =							modular_exponentiation(lowercase							,      n - 2							)
				return fa * matrix.t[0][0] + fa * matrix.t[0][1]





def     UpperCAmelCase						(   lowercase							,      lowercase							,      lowercase							):







				"""simple docstring"""



				if n == 1:
								return fa
				elif n == 2:
								return fa

				__lowercase			      =							fa
				__lowercase			      =							fa
				n -= 2

				while n > 0:
								__lowercase      ,					__lowercase			      =							fn_a + fn_a, fn_a
								n -= 1

				return fn_a





def     UpperCAmelCase						(   ):







				"""simple docstring"""



				__lowercase			      =							'''
from random import randint
from __main__ import fibonacci_with_matrix_exponentiation
'''
				__lowercase			      =							'''fibonacci_with_matrix_exponentiation(randint(1,70000), 1, 1)'''
				__lowercase			      =							timeit.timeit(setup=lowercase							,      stmt=lowercase							,      number=100							)
				print('''With matrix exponentiation the average execution time is '''							,      exec_time / 100							)
				return exec_time





def     UpperCAmelCase						(   ):







				"""simple docstring"""



				__lowercase			      =							'''
from random import randint
from __main__ import simple_fibonacci
'''
				__lowercase			      =							'''simple_fibonacci(randint(1,70000), 1, 1)'''
				__lowercase			      =							timeit.timeit(setup=lowercase							,      stmt=lowercase							,      number=100							)
				print(
				    '''Without matrix exponentiation the average execution time is '''							,      exec_time / 100							)
				return exec_time





def     UpperCAmelCase						(   ):







				"""simple docstring"""



				matrix_exponentiation_time()
				simple_fibonacci_time()


if __name__ == "__main__":
		main()