


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 			snake_case__(							UpperCAmelCase__,  unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__ :					Dict   =							PreTrainedTokenizerFast
						SCREAMING_SNAKE_CASE__ :					List[Any]   =							False
						SCREAMING_SNAKE_CASE__ :					Union[str, Any]   =							True
						SCREAMING_SNAKE_CASE__ :					Dict   =							"""tokenizer_file"""




						def   lowercase_	(			self							)   ->      Optional[int]:
										lowerCAmelCase_	: Any      			=			False  # because we don't have pretrained_vocab_files_map
										super().setUp()
										lowerCAmelCase_	: Optional[int]      			=			True

										lowerCAmelCase_	: int      			=			['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
										lowerCAmelCase_	: Tuple      			=			'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

										# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
										lowerCAmelCase_	: Tuple      			=			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

										lowerCAmelCase_	: str      			=			PreTrainedTokenizerFast.from_pretrained(model_paths[0]							)
										tokenizer.save_pretrained(self.tmpdirname							)




						def   lowercase_	(			self							)   ->      Dict:
										# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
										# model
										pass




						def   lowercase_	(			self							)   ->      Dict:
										# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
										# model
										pass




						def   lowercase_	(			self							)   ->      int:
										# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
										# model
										pass




						def   lowercase_	(			self							)   ->      List[Any]:
										# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
										pass




						def   lowercase_	(			self							)   ->      List[Any]:
										lowerCAmelCase_	: Optional[int]      			=			self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																		try:
																						lowerCAmelCase_	: List[str]      			=			tempfile.mkdtemp()
																						lowerCAmelCase_	: List[str]      			=			self.rust_tokenizer_class.from_pretrained(__lowercase				,  **__lowercase							)

																						tokenizer.save_pretrained(self.tmpdirname							)
																						super().test_training_new_tokenizer()
																		finally:
																						# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																						# is restored
																						shutil.rmtree(self.tmpdirname							)
																						lowerCAmelCase_	: Union[str, Any]      			=			tmpdirname_orig




						def   lowercase_	(			self							)   ->      Optional[int]:
										lowerCAmelCase_	: List[Any]      			=			self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																		try:
																						lowerCAmelCase_	: Any      			=			tempfile.mkdtemp()
																						lowerCAmelCase_	: List[Any]      			=			self.rust_tokenizer_class.from_pretrained(__lowercase				,  **__lowercase							)

																						tokenizer.save_pretrained(self.tmpdirname							)
																						super().test_training_new_tokenizer_with_special_tokens_change()
																		finally:
																						# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																						# is restored
																						shutil.rmtree(self.tmpdirname							)
																						lowerCAmelCase_	: List[Any]      			=			tmpdirname_orig




						def   lowercase_	(			self							)   ->      List[Any]:
										lowerCAmelCase_	: Optional[int]      			=			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name							)

										lowerCAmelCase_	: int      			=			('''a''' for _ in range(1_0_0_0							))
										lowerCAmelCase_	: Dict      			=			tokenizer.train_new_from_iterator(text_iterator=__lowercase				,  length=1_0_0_0				,  vocab_size=5_0							)

										lowerCAmelCase_	: List[Any]      			=			new_tokenizer.encode('''a🤗'''							)
										self.assertEqual(__lowercase				,  [6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]							)







@require_tokenizers
class 			snake_case__(							unittest.TestCase							):







						'''simple docstring'''




						def   lowercase_	(			self							)   ->      Optional[Any]:
										lowerCAmelCase_	: Tuple      			=			AutoTokenizer.from_pretrained('''bert-base-cased'''							)
										lowerCAmelCase_	: List[str]      			=			json.loads(tokenizer._tokenizer.to_str()							)
										lowerCAmelCase_	: List[str]      			=			len(__lowercase							)

										with tempfile.TemporaryDirectory() as tmp_dir:
														# Hack to save this in the tokenizer_config.json
														lowerCAmelCase_	: Optional[int]      			=			['''tokenizer.4.0.0.json''']
														tokenizer.save_pretrained(__lowercase							)
														json.dump(__lowercase				,  open(os.path.join(__lowercase				,  '''tokenizer.4.0.0.json'''							)				,  '''w'''							)							)

														# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
														lowerCAmelCase_	: int      			=			AutoTokenizer.from_pretrained(__lowercase							)
														self.assertEqual(len(__lowercase							)				,  len(__lowercase							) + 1							)
														lowerCAmelCase_	: List[Any]      			=			json.loads(new_tokenizer._tokenizer.to_str()							)
														self.assertIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']							)

														# Will need to be adjusted if we reach v42 and this test is still here.
														# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
														shutil.move(os.path.join(__lowercase				,  '''tokenizer.4.0.0.json'''							)				,  os.path.join(__lowercase				,  '''tokenizer.42.0.0.json'''							)							)
														lowerCAmelCase_	: Dict      			=			['''tokenizer.42.0.0.json''']
														tokenizer.save_pretrained(__lowercase							)
														lowerCAmelCase_	: Any      			=			AutoTokenizer.from_pretrained(__lowercase							)
														self.assertEqual(len(__lowercase							)				,  len(__lowercase							)							)
														lowerCAmelCase_	: Optional[Any]      			=			json.loads(new_tokenizer._tokenizer.to_str()							)
														self.assertNotIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']							)




						def   lowercase_	(			self							)   ->      int:
										# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
										lowerCAmelCase_	: List[Any]      			=			'''hf-internal-testing/test-two-tokenizers'''

										# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
										lowerCAmelCase_	: List[Any]      			=			AutoTokenizer.from_pretrained(__lowercase							)
										self.assertEqual(len(__lowercase							)				,  2_8_9_9_7							)
										lowerCAmelCase_	: Optional[Any]      			=			json.loads(tokenizer._tokenizer.to_str()							)
										self.assertIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']							)

										# Testing an older version by monkey-patching the version in the module it's used.
										import transformers as old_transformers

										lowerCAmelCase_	: Dict      			=			'''3.0.0'''
										lowerCAmelCase_	: Union[str, Any]      			=			old_transformers.models.auto.AutoTokenizer.from_pretrained(__lowercase							)
										self.assertEqual(len(__lowercase							)				,  2_8_9_9_6							)
										lowerCAmelCase_	: int      			=			json.loads(old_tokenizer._tokenizer.to_str()							)
										self.assertNotIn('''huggingface'''				,  json_tokenizer['''model''']['''vocab''']							)







@require_tokenizers
class 			snake_case__(							unittest.TestCase							):







						'''simple docstring'''




						def   lowercase_	(			self							)   ->      Tuple:
										# See https://github.com/huggingface/transformers/pull/12550
										# and https://github.com/huggingface/tokenizers/issues/537
										lowerCAmelCase_	: str      			=			PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''							)
										lowerCAmelCase_	: Any      			=			'''The Matrix is a 1999 science fiction action film.'''

										with concurrent.futures.ThreadPoolExecutor() as executor:
														lowerCAmelCase_	: List[str]      			=			[executor.submit(self.fetch				,  __lowercase				,  __lowercase							) for i in range(1_0							)]
														lowerCAmelCase_	: str      			=			[future.result() for future in futures]
														self.assertEqual(__lowercase				,  [[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0							)]							)




						def   lowercase_	(			self				,  __lowercase				,  __lowercase							)   ->      str:
										return tokenizer.encode(__lowercase				,  truncation='''longest_first'''				,  padding='''longest'''							)



from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase     :					List[str]       =logging.get_logger(__name__)







class 			snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__ :					List[str]   =							["""input_ids""", """attention_mask"""]




						def __init__(			self				,  __lowercase="[PAD]"				,  __lowercase="[BOS]"				,  __lowercase="[EOS]"				,  __lowercase="[MASK]"				,  __lowercase="[CLS]"				,  __lowercase="[SEP]"				,  __lowercase=2_0_4_8				,  **__lowercase				,  )   ->      None:
										lowerCAmelCase_	: Optional[int]      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else pad_token
										lowerCAmelCase_	: str      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else bos_token
										lowerCAmelCase_	: Dict      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else eos_token
										lowerCAmelCase_	: List[str]      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else mask_token
										lowerCAmelCase_	: Dict      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else cls_token
										lowerCAmelCase_	: Dict      			=			AddedToken(__lowercase				,  lstrip=__lowercase				,  rstrip=__lowercase							) if isinstance(__lowercase				,  __lowercase							) else sep_token

										super().__init__(
										    pad_token=__lowercase				,  bos_token=__lowercase				,  eos_token=__lowercase				,  mask_token=__lowercase				,  cls_token=__lowercase				,  sep_token=__lowercase				,  model_max_length=__lowercase				,  **__lowercase				,  )

										lowerCAmelCase_	: Dict      			=			2**8  # utf is 8 bits

										# define special tokens dict
										lowerCAmelCase_	: Dict[str, int]      			=			{
										    self.pad_token: 0,
										    self.bos_token: 1,
										    self.eos_token: 2,
										    self.mask_token: 3,
										    self.cls_token: 4,
										    self.sep_token: 5,
										}
										lowerCAmelCase_	: List[Any]      			=			len(self.special_tokens_encoder							)
										lowerCAmelCase_	: Dict[int, str]      			=			{v: k for k, v in self.special_tokens_encoder.items()}




						def   lowercase_	(			self							)   ->      Dict[str, int]:
										lowerCAmelCase_	: int      			=			self.special_tokens_encoder.copy()
										vocab.update(self.added_tokens_encoder							)
										for i in range(self._utf_vocab_size							):
														lowerCAmelCase_	: int      			=			chr(__lowercase							)
														lowerCAmelCase_	: Optional[Any]      			=			i + len(self.special_tokens_encoder							)
										return vocab




						@property
						def   lowercase_	(			self							)   ->      str:
										return self._utf_vocab_size + self._num_special_tokens




						def   lowercase_	(			self				,  __lowercase				,  __lowercase = None				,  __lowercase = False							)   ->      List[int]:
										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=__lowercase				,  token_ids_a=__lowercase				,  already_has_special_tokens=__lowercase							)

										# normal case: some special tokens
										if token_ids_a is None:
														return [1] + [0] * len(__lowercase							) + [1]
										return [1] + ([0] * len(__lowercase							)) + [1] + ([0] * len(__lowercase							)) + [1]




						def   lowercase_	(			self				,  __lowercase				,  __lowercase = None							)   ->      List[int]:
										if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										else:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]




						def   lowercase_	(			self				,  __lowercase							)   ->      List[str]:
										lowerCAmelCase_	: Optional[int]      			=			[chr(__lowercase							) for i in text.encode('''utf-8'''							)]
										return tokens




						def   lowercase_	(			self				,  __lowercase							)   ->      Tuple:
										if token in self.special_tokens_encoder:
														lowerCAmelCase_	: Optional[int]      			=			self.special_tokens_encoder[token]
										elif token in self.added_tokens_encoder:
														lowerCAmelCase_	: Any      			=			self.added_tokens_encoder[token]
										elif len(__lowercase							) != 1:
														lowerCAmelCase_	: Any      			=			self.unk_token_id
										else:
														lowerCAmelCase_	: List[Any]      			=			ord(__lowercase							) + self._num_special_tokens
										return token_id




						def   lowercase_	(			self				,  __lowercase							)   ->      Optional[int]:
										if index in self.special_tokens_decoder:
														lowerCAmelCase_	: List[Any]      			=			self.special_tokens_decoder[index]
										elif index in self.added_tokens_decoder:
														lowerCAmelCase_	: List[str]      			=			self.added_tokens_decoder[index]
										else:
														lowerCAmelCase_	: str      			=			chr(index - self._num_special_tokens							)
										return token




						def   lowercase_	(			self				,  __lowercase							)   ->      Optional[int]:
										lowerCAmelCase_	: Tuple      			=			B''''''
										for token in tokens:
														if token in self.special_tokens_decoder:
																		lowerCAmelCase_	: int      			=			self.special_tokens_decoder[token].encode('''utf-8'''							)
														elif token in self.added_tokens_decoder:
																		lowerCAmelCase_	: Tuple      			=			self.special_tokens_decoder[token].encode('''utf-8'''							)
														elif token in self.special_tokens_encoder:
																		lowerCAmelCase_	: Any      			=			token.encode('''utf-8'''							)
														elif token in self.added_tokens_encoder:
																		lowerCAmelCase_	: Tuple      			=			token.encode('''utf-8'''							)
														else:
																		lowerCAmelCase_	: str      			=			bytes([ord(__lowercase							)]							)
														bstring += tok_string
										lowerCAmelCase_	: str      			=			bstring.decode('''utf-8'''				,  errors='''replace'''							)
										return string




						def   lowercase_	(			self				,  __lowercase				,  __lowercase = None							)   ->      Tuple[str]:
										return ()