



import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece





class  __lowerCAmelCase			(					unittest.TestCase	):




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   list(tokenizer_class.max_model_input_sizes.keys()		)
									for model_name in sa_models[:1]:
														_lowerCAmelCase      =   tokenizer_class.from_pretrained(_snake_case		)
														self.assertIsNotNone(_snake_case		)
														self.assertIsInstance(_snake_case						,    _snake_case		)
														self.assertIsInstance(_snake_case						,    _snake_case		)

														for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(_snake_case						,    _snake_case		)
																			_lowerCAmelCase      =   tokenizer.convert_tokens_to_ids(_snake_case		)
																			self.assertIsInstance(_snake_case						,    _snake_case		)




				def 							snake_case       (							self						,    _snake_case						,    _snake_case = None		):



									"""simple docstring"""





									_lowerCAmelCase      =   pickle.dumps(_snake_case		)
									self.assertIsNotNone(_snake_case		)

									_lowerCAmelCase      =   pickle.loads(_snake_case		)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast						,    be_original.is_fast		)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
														self.assertIsNotNone(be_restored.encodings		)
									else:
														self.assertIsNone(be_restored.encodings		)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()						,    be_restored.values()		):
														if equal_op:
																			self.assertTrue(equal_op(_snake_case						,    _snake_case		)		)
														else:
																			self.assertEqual(_snake_case						,    _snake_case		)




				@slow
				def 							snake_case       (							self		):



									"""simple docstring"""





									self.check_tokenizer_from_pretrained(_snake_case		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									self.assertEqual(TensorType("""tf"""		)						,    TensorType.TENSORFLOW		)
									self.assertEqual(TensorType("""pt"""		)						,    TensorType.PYTORCH		)
									self.assertEqual(TensorType("""np"""		)						,    TensorType.NUMPY		)




				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									import numpy as np

									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)
									_lowerCAmelCase      =   BertTokenizerFast.from_pretrained("""bert-base-cased"""		)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""		):
														self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""		)		)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""		):
														self.assert_dump_and_restore(
														    tokenizer_p("""Small example to encode"""						,    return_tensors=TensorType.NUMPY		)						,    np.array_equal		)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""		):
														self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""		)		)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""		):
														self.assert_dump_and_restore(
														    tokenizer_r("""Small example to encode"""						,    return_tensors=TensorType.NUMPY		)						,    np.array_equal		)




				@require_tf
				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									import tensorflow as tf

									def tf_array_equals(_snake_case						,    _snake_case		):
														return tf.reduce_all(tf.equal(_snake_case						,    _snake_case		)		)

									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)
									_lowerCAmelCase      =   BertTokenizerFast.from_pretrained("""bert-base-cased"""		)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""		):
														self.assert_dump_and_restore(
														    tokenizer_p("""Small example to encode"""						,    return_tensors=TensorType.TENSORFLOW		)						,    _snake_case		)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""		):
														self.assert_dump_and_restore(
														    tokenizer_r("""Small example to encode"""						,    return_tensors=TensorType.TENSORFLOW		)						,    _snake_case		)




				@require_torch
				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									import torch

									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)
									_lowerCAmelCase      =   BertTokenizerFast.from_pretrained("""bert-base-cased"""		)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""		):
														self.assert_dump_and_restore(
														    tokenizer_p("""Small example to encode"""						,    return_tensors=TensorType.PYTORCH		)						,    torch.equal		)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""		):
														self.assert_dump_and_restore(
														    tokenizer_r("""Small example to encode"""						,    return_tensors=TensorType.PYTORCH		)						,    torch.equal		)




				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)
									_lowerCAmelCase      =   BertTokenizerFast.from_pretrained("""bert-base-cased"""		)

									with self.subTest("""Python Tokenizer"""		):
														self.assertFalse(tokenizer_p("""Small example to_encode"""		).is_fast		)

									with self.subTest("""Rust Tokenizer"""		):
														self.assertTrue(tokenizer_r("""Small example to_encode"""		).is_fast		)




				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   BertTokenizerFast.from_pretrained("""bert-base-cased"""		)
									_lowerCAmelCase      =   tokenizer_r(["""Test""", """\xad""", """test"""]						,    is_split_into_words=_snake_case		)

									self.assertEqual(encoded.word_to_tokens(0		)						,    TokenSpan(start=1						,    end=2		)		)
									self.assertEqual(encoded.word_to_tokens(1		)						,    _snake_case		)
									self.assertEqual(encoded.word_to_tokens(2		)						,    TokenSpan(start=2						,    end=3		)		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""np"""		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (2, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (2,)		)
									# test converting the converted
									with CaptureStderr() as cs:
														_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""np"""		)
									self.assertFalse(len(cs.err		)						,    msg=F'should have no warning, but got {cs.err}'		)

									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""np"""						,    prepend_batch_axis=_snake_case		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (1, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (1,)		)




				@require_torch
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""pt"""		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (2, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (2,)		)
									# test converting the converted
									with CaptureStderr() as cs:
														_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""pt"""		)
									self.assertFalse(len(cs.err		)						,    msg=F'should have no warning, but got {cs.err}'		)

									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""pt"""						,    prepend_batch_axis=_snake_case		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (1, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (1,)		)




				@require_tf
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""tf"""		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (2, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (2,)		)
									# test converting the converted
									with CaptureStderr() as cs:
														_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""tf"""		)
									self.assertFalse(len(cs.err		)						,    msg=F'should have no warning, but got {cs.err}'		)

									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""tf"""						,    prepend_batch_axis=_snake_case		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (1, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (1,)		)




				@require_flax
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""jax"""		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (2, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (2,)		)
									# test converting the converted
									with CaptureStderr() as cs:
														_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""jax"""		)
									self.assertFalse(len(cs.err		)						,    msg=F'should have no warning, but got {cs.err}'		)

									_lowerCAmelCase      =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}		)
									_lowerCAmelCase      =   batch.convert_to_tensors(tensor_type="""jax"""						,    prepend_batch_axis=_snake_case		)
									self.assertEqual(tensor_batch["""inputs"""].shape						,    (1, 3)		)
									self.assertEqual(tensor_batch["""labels"""].shape						,    (1,)		)




				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   [{"""input_ids""": np.array([0, 1, 2]		)}, {"""input_ids""": np.array([0, 1, 2, 3]		)}]
									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)

									_lowerCAmelCase      =   tokenizer.pad(_snake_case						,    padding=_snake_case		)
									self.assertTrue(isinstance(batch["""input_ids"""]						,    np.ndarray		)		)
									self.assertEqual(batch["""input_ids"""].tolist()						,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
									_lowerCAmelCase      =   tokenizer.pad(_snake_case						,    padding=_snake_case						,    return_tensors="""np"""		)
									self.assertTrue(isinstance(batch["""input_ids"""]						,    np.ndarray		)		)
									self.assertEqual(batch["""input_ids"""].tolist()						,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)




				@require_torch
				def 							snake_case       (							self		):



									"""simple docstring"""





									import torch

									_lowerCAmelCase      =   [{"""input_ids""": torch.tensor([0, 1, 2]		)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]		)}]
									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)

									_lowerCAmelCase      =   tokenizer.pad(_snake_case						,    padding=_snake_case		)
									self.assertTrue(isinstance(batch["""input_ids"""]						,    torch.Tensor		)		)
									self.assertEqual(batch["""input_ids"""].tolist()						,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
									_lowerCAmelCase      =   tokenizer.pad(_snake_case						,    padding=_snake_case						,    return_tensors="""pt"""		)
									self.assertTrue(isinstance(batch["""input_ids"""]						,    torch.Tensor		)		)
									self.assertEqual(batch["""input_ids"""].tolist()						,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)




				@require_tf
				def 							snake_case       (							self		):



									"""simple docstring"""





									import tensorflow as tf

									_lowerCAmelCase      =   [{"""input_ids""": tf.constant([0, 1, 2]		)}, {"""input_ids""": tf.constant([0, 1, 2, 3]		)}]
									_lowerCAmelCase      =   BertTokenizer.from_pretrained("""bert-base-cased"""		)

									_lowerCAmelCase      =   tokenizer.pad(_snake_case						,    padding=_snake_case		)
									self.assertTrue(isinstance(batch["""input_ids"""]						,    tf.Tensor		)		)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()						,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
									_lowerCAmelCase      =   tokenizer.pad(_snake_case						,    padding=_snake_case						,    return_tensors="""tf"""		)
									self.assertTrue(isinstance(batch["""input_ids"""]						,    tf.Tensor		)		)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()						,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)




				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   Tokenizer(WordPiece(unk_token="""[UNK]"""		)		)
									PreTrainedTokenizerFast(tokenizer_object=_snake_case		)




				@require_tokenizers
				def 							snake_case       (							self		):



									"""simple docstring"""





									_lowerCAmelCase      =   Tokenizer(WordPiece(unk_token="""[UNK]"""		)		)
									with tempfile.TemporaryDirectory() as tmpdirname:
														bert_tokenizer.save(os.path.join(_snake_case						,    """tokenizer.json"""		)		)
														PreTrainedTokenizerFast(tokenizer_file=os.path.join(_snake_case						,    """tokenizer.json"""		)		)








# Lint as: python3

import hashlib
from typing import Union


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					if isinstance(snake_case     ,						snake_case      ):
										# Data already in bytes, returns as it as
										return hash_data
					elif isinstance(snake_case     ,						snake_case      ):
										# We keep the data as it as for it ot be later encoded to UTF-8
										# However replace `\\` with `/` for Windows compatibility
										_lowerCAmelCase      =   hash_data.replace("""\\"""     ,						"""/"""      )
					elif isinstance(snake_case     ,						snake_case      ):
										_lowerCAmelCase      =   str(snake_case      )
					else:
										# If data is not of the required type, raise error
										raise InvalidKeyError(snake_case      )

					return hash_data.encode("""utf-8"""      )





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   """\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
									_lowerCAmelCase      =   F'\nFound Key {hash_data} of type {type(_snake_case		)}'
									_lowerCAmelCase      =   """\nKeys should be either str, int or bytes type"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case						,    _snake_case						,    _snake_case=""		):



									"""simple docstring"""





									_lowerCAmelCase      =   key
									_lowerCAmelCase      =   duplicate_key_indices
									_lowerCAmelCase      =   fix_msg
									_lowerCAmelCase      =   """Found multiple examples generated with the same key"""
									if len(_snake_case		) <= 20:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(_snake_case		)} have the key {key}'
									else:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(duplicate_key_indices[:20]		)}... ({len(_snake_case		) - 20} more) have the key {key}'
									_lowerCAmelCase      =   """\n""" + fix_msg if fix_msg else """"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			:




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   hashlib.mda(_as_bytes(_snake_case		)		)




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   self._split_mda.copy()
									_lowerCAmelCase      =   _as_bytes(_snake_case		)
									mda.update(_snake_case		)
									# Convert to integer with hexadecimal conversion
									return int(mda.hexdigest()						,    16		)



