




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece







class 				UpperCAmelCase(					unittest.TestCase					):



	"""simple docstring"""






	def    __a     (					self	,				lowerCamelCase    ) ->      List[Any]:





								"""simple docstring"""


								lowercase__  :	int        =							list(tokenizer_class.max_model_input_sizes.keys()    )
								for model_name in sa_models[:1]:
															lowercase__  :	List[Any]        =							tokenizer_class.from_pretrained(snake_case_    )
															self.assertIsNotNone(snake_case_    )
															self.assertIsInstance(snake_case_	,				snake_case_    )
															self.assertIsInstance(snake_case_	,				snake_case_    )

															for special_tok in tokenizer.all_special_tokens:
																						self.assertIsInstance(snake_case_	,				snake_case_    )
																						lowercase__  :	Any        =							tokenizer.convert_tokens_to_ids(snake_case_    )
																						self.assertIsInstance(snake_case_	,				snake_case_    )

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      Optional[Any]:





								"""simple docstring"""


								lowercase__  :	Dict        =							pickle.dumps(snake_case_    )
								self.assertIsNotNone(snake_case_    )

								lowercase__  :	List[str]        =							pickle.loads(snake_case_    )

								# Ensure is_fast is correctly restored
								self.assertEqual(be_restored.is_fast	,				be_original.is_fast    )

								# Ensure encodings are potentially correctly restored
								if be_original.is_fast:
															self.assertIsNotNone(be_restored.encodings    )
								else:
															self.assertIsNone(be_restored.encodings    )

								# Ensure the keys are the same
								for original_v, restored_v in zip(be_original.values()	,				be_restored.values()    ):
															if equal_op:
																						self.assertTrue(equal_op(snake_case_	,				snake_case_    )    )
															else:
																						self.assertEqual(snake_case_	,				snake_case_    )

	@slow
	def    __a     (					self    ) ->      Tuple:





								"""simple docstring"""


								self.check_tokenizer_from_pretrained(snake_case_    )

	def    __a     (					self    ) ->      Tuple:





								"""simple docstring"""


								self.assertEqual(TensorType("tf"    )	,				TensorType.TENSORFLOW    )
								self.assertEqual(TensorType("pt"    )	,				TensorType.PYTORCH    )
								self.assertEqual(TensorType("np"    )	,				TensorType.NUMPY    )

	@require_tokenizers
	def    __a     (					self    ) ->      str:





								"""simple docstring"""


								import numpy as np

								lowercase__  :	Optional[int]        =							BertTokenizer.from_pretrained("bert-base-cased"    )
								lowercase__  :	Dict        =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

								# Python no tensor
								with self.subTest("BatchEncoding (Python, return_tensors=None)"    ):
															self.assert_dump_and_restore(tokenizer_p("Small example to encode"    )    )

								with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"    ):
															self.assert_dump_and_restore(
															    tokenizer_p("Small example to encode"	,				return_tensors=TensorType.NUMPY    )	,				np.array_equal    )

								with self.subTest("BatchEncoding (Rust, return_tensors=None)"    ):
															self.assert_dump_and_restore(tokenizer_r("Small example to encode"    )    )

								with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"    ):
															self.assert_dump_and_restore(
															    tokenizer_r("Small example to encode"	,				return_tensors=TensorType.NUMPY    )	,				np.array_equal    )

	@require_tf
	@require_tokenizers
	def    __a     (					self    ) ->      Optional[Any]:





								"""simple docstring"""


								import tensorflow as tf

								def tf_array_equals(lowerCamelCase	,				lowerCamelCase    ):
															return tf.reduce_all(tf.equal(snake_case_	,				snake_case_    )    )

								lowercase__  :	str        =							BertTokenizer.from_pretrained("bert-base-cased"    )
								lowercase__  :	List[Any]        =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

								with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"    ):
															self.assert_dump_and_restore(
															    tokenizer_p("Small example to encode"	,				return_tensors=TensorType.TENSORFLOW    )	,				snake_case_    )

								with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"    ):
															self.assert_dump_and_restore(
															    tokenizer_r("Small example to encode"	,				return_tensors=TensorType.TENSORFLOW    )	,				snake_case_    )

	@require_torch
	@require_tokenizers
	def    __a     (					self    ) ->      Optional[int]:





								"""simple docstring"""


								import torch

								lowercase__  :	Optional[int]        =							BertTokenizer.from_pretrained("bert-base-cased"    )
								lowercase__  :	Optional[Any]        =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

								with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"    ):
															self.assert_dump_and_restore(
															    tokenizer_p("Small example to encode"	,				return_tensors=TensorType.PYTORCH    )	,				torch.equal    )

								with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"    ):
															self.assert_dump_and_restore(
															    tokenizer_r("Small example to encode"	,				return_tensors=TensorType.PYTORCH    )	,				torch.equal    )

	@require_tokenizers
	def    __a     (					self    ) ->      int:





								"""simple docstring"""


								lowercase__  :	Tuple        =							BertTokenizer.from_pretrained("bert-base-cased"    )
								lowercase__  :	List[str]        =							BertTokenizerFast.from_pretrained("bert-base-cased"    )

								with self.subTest("Python Tokenizer"    ):
															self.assertFalse(tokenizer_p("Small example to_encode"    ).is_fast    )

								with self.subTest("Rust Tokenizer"    ):
															self.assertTrue(tokenizer_r("Small example to_encode"    ).is_fast    )

	@require_tokenizers
	def    __a     (					self    ) ->      int:





								"""simple docstring"""


								lowercase__  :	Union[str, Any]        =							BertTokenizerFast.from_pretrained("bert-base-cased"    )
								lowercase__  :	Tuple        =							tokenizer_r(["Test", "\xad", "test"]	,				is_split_into_words=snake_case_    )

								self.assertEqual(encoded.word_to_tokens(0    )	,				TokenSpan(start=1	,				end=2    )    )
								self.assertEqual(encoded.word_to_tokens(1    )	,				snake_case_    )
								self.assertEqual(encoded.word_to_tokens(2    )	,				TokenSpan(start=2	,				end=3    )    )

	def    __a     (					self    ) ->      List[str]:





								"""simple docstring"""


								lowercase__  :	Any        =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
								lowercase__  :	Union[str, Any]        =							batch.convert_to_tensors(tensor_type="np"    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(2, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(2,)    )
								# test converting the converted
								with CaptureStderr() as cs:
															lowercase__  :	List[str]        =							batch.convert_to_tensors(tensor_type="np"    )
								self.assertFalse(len(cs.err    )	,				msg=f"""should have no warning, but got {cs.err}"""    )

								lowercase__  :	Dict        =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
								lowercase__  :	List[str]        =							batch.convert_to_tensors(tensor_type="np"	,				prepend_batch_axis=snake_case_    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(1, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(1,)    )

	@require_torch
	def    __a     (					self    ) ->      int:





								"""simple docstring"""


								lowercase__  :	Tuple        =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
								lowercase__  :	Optional[Any]        =							batch.convert_to_tensors(tensor_type="pt"    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(2, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(2,)    )
								# test converting the converted
								with CaptureStderr() as cs:
															lowercase__  :	Union[str, Any]        =							batch.convert_to_tensors(tensor_type="pt"    )
								self.assertFalse(len(cs.err    )	,				msg=f"""should have no warning, but got {cs.err}"""    )

								lowercase__  :	str        =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
								lowercase__  :	List[Any]        =							batch.convert_to_tensors(tensor_type="pt"	,				prepend_batch_axis=snake_case_    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(1, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(1,)    )

	@require_tf
	def    __a     (					self    ) ->      Any:





								"""simple docstring"""


								lowercase__  :	Dict        =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
								lowercase__  :	List[str]        =							batch.convert_to_tensors(tensor_type="tf"    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(2, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(2,)    )
								# test converting the converted
								with CaptureStderr() as cs:
															lowercase__  :	Union[str, Any]        =							batch.convert_to_tensors(tensor_type="tf"    )
								self.assertFalse(len(cs.err    )	,				msg=f"""should have no warning, but got {cs.err}"""    )

								lowercase__  :	str        =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
								lowercase__  :	Any        =							batch.convert_to_tensors(tensor_type="tf"	,				prepend_batch_axis=snake_case_    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(1, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(1,)    )

	@require_flax
	def    __a     (					self    ) ->      Union[str, Any]:





								"""simple docstring"""


								lowercase__  :	Tuple        =							BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}    )
								lowercase__  :	Tuple        =							batch.convert_to_tensors(tensor_type="jax"    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(2, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(2,)    )
								# test converting the converted
								with CaptureStderr() as cs:
															lowercase__  :	str        =							batch.convert_to_tensors(tensor_type="jax"    )
								self.assertFalse(len(cs.err    )	,				msg=f"""should have no warning, but got {cs.err}"""    )

								lowercase__  :	str        =							BatchEncoding({"inputs": [1, 2, 3], "labels": 0}    )
								lowercase__  :	List[str]        =							batch.convert_to_tensors(tensor_type="jax"	,				prepend_batch_axis=snake_case_    )
								self.assertEqual(tensor_batch["inputs"].shape	,				(1, 3)    )
								self.assertEqual(tensor_batch["labels"].shape	,				(1,)    )

	def    __a     (					self    ) ->      Dict:





								"""simple docstring"""


								lowercase__  :	Optional[Any]        =							[{"input_ids": np.array([0, 1, 2]    )}, {"input_ids": np.array([0, 1, 2, 3]    )}]
								lowercase__  :	Optional[Any]        =							BertTokenizer.from_pretrained("bert-base-cased"    )

								lowercase__  :	List[str]        =							tokenizer.pad(snake_case_	,				padding=snake_case_    )
								self.assertTrue(isinstance(batch["input_ids"]	,				np.ndarray    )    )
								self.assertEqual(batch["input_ids"].tolist()	,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
								lowercase__  :	str        =							tokenizer.pad(snake_case_	,				padding=snake_case_	,				return_tensors="np"    )
								self.assertTrue(isinstance(batch["input_ids"]	,				np.ndarray    )    )
								self.assertEqual(batch["input_ids"].tolist()	,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

	@require_torch
	def    __a     (					self    ) ->      Optional[int]:





								"""simple docstring"""


								import torch

								lowercase__  :	Union[str, Any]        =							[{"input_ids": torch.tensor([0, 1, 2]    )}, {"input_ids": torch.tensor([0, 1, 2, 3]    )}]
								lowercase__  :	Tuple        =							BertTokenizer.from_pretrained("bert-base-cased"    )

								lowercase__  :	int        =							tokenizer.pad(snake_case_	,				padding=snake_case_    )
								self.assertTrue(isinstance(batch["input_ids"]	,				torch.Tensor    )    )
								self.assertEqual(batch["input_ids"].tolist()	,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
								lowercase__  :	List[Any]        =							tokenizer.pad(snake_case_	,				padding=snake_case_	,				return_tensors="pt"    )
								self.assertTrue(isinstance(batch["input_ids"]	,				torch.Tensor    )    )
								self.assertEqual(batch["input_ids"].tolist()	,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

	@require_tf
	def    __a     (					self    ) ->      Dict:





								"""simple docstring"""


								import tensorflow as tf

								lowercase__  :	Dict        =							[{"input_ids": tf.constant([0, 1, 2]    )}, {"input_ids": tf.constant([0, 1, 2, 3]    )}]
								lowercase__  :	int        =							BertTokenizer.from_pretrained("bert-base-cased"    )

								lowercase__  :	Any        =							tokenizer.pad(snake_case_	,				padding=snake_case_    )
								self.assertTrue(isinstance(batch["input_ids"]	,				tf.Tensor    )    )
								self.assertEqual(batch["input_ids"].numpy().tolist()	,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
								lowercase__  :	Tuple        =							tokenizer.pad(snake_case_	,				padding=snake_case_	,				return_tensors="tf"    )
								self.assertTrue(isinstance(batch["input_ids"]	,				tf.Tensor    )    )
								self.assertEqual(batch["input_ids"].numpy().tolist()	,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

	@require_tokenizers
	def    __a     (					self    ) ->      str:





								"""simple docstring"""


								lowercase__  :	Tuple        =							Tokenizer(WordPiece(unk_token="[UNK]"    )    )
								PreTrainedTokenizerFast(tokenizer_object=snake_case_    )







	@require_tokenizers
	def    __a     (					self    ) ->      List[str]:





								"""simple docstring"""


								lowercase__  :	List[Any]        =							Tokenizer(WordPiece(unk_token="[UNK]"    )    )
								with tempfile.TemporaryDirectory() as tmpdirname:
															bert_tokenizer.save(os.path.join(snake_case_	,				"tokenizer.json"    )    )
															PreTrainedTokenizerFast(tokenizer_file=os.path.join(snake_case_	,				"tokenizer.json"    )    )





def  snake_case_					(		SCREAMING_SNAKE_CASE_    )      ->			list:
							lowercase__  :	Optional[Any]        =							len(SCREAMING_SNAKE_CASE_    )

							# Set the gap value to be decreased by a factor of 1.3
							# after each iteration
							lowercase__  :	Tuple        =							1.3

							# Continue sorting until the gap is 1
							while gap > 1:
														# Decrease the gap value
														lowercase__  :	Tuple        =							int(gap / shrink    )

														# Sort the elements using insertion sort
														for i in range(SCREAMING_SNAKE_CASE_    ,len(SCREAMING_SNAKE_CASE_    )    ):
																					lowercase__  :	int        =							collection[i]
																					lowercase__  :	Union[str, Any]        =							i
																					while j >= gap and collection[j - gap] > temp:
																												lowercase__  :	str        =							collection[j - gap]
																												j -= gap
																					lowercase__  :	List[Any]        =							temp

							return collection


if __name__ == "__main__":
						import doctest

						doctest.testmod()