




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece






class      A     (						unittest.TestCase						):
     """simple docstring"""

     def 	__snake_case				(      self					:				Optional[int]		,					A__					:				int     )			->    Optional[int]:
      lowerCamelCase__ 					=			list(tokenizer_class.max_model_input_sizes.keys()     )
      for model_name in sa_models[:1]:
       lowerCamelCase__ 					=			tokenizer_class.from_pretrained(__lowerCAmelCase     )
       self.assertIsNotNone(__lowerCAmelCase     )
       self.assertIsInstance(__lowerCAmelCase		,					__lowerCAmelCase     )
       self.assertIsInstance(__lowerCAmelCase		,					__lowerCAmelCase     )

       for special_tok in tokenizer.all_special_tokens:
        self.assertIsInstance(__lowerCAmelCase		,					__lowerCAmelCase     )
        lowerCamelCase__ 					=			tokenizer.convert_tokens_to_ids(__lowerCAmelCase     )
        self.assertIsInstance(__lowerCAmelCase		,					__lowerCAmelCase     )




     def 	__snake_case				(      self					:				List[str]		,					A__					:				str		,					A__					:				Union[str, Any] = None     )			->    int:
      lowerCamelCase__ 					=			pickle.dumps(__lowerCAmelCase     )
      self.assertIsNotNone(__lowerCAmelCase     )

      lowerCamelCase__ 					=			pickle.loads(__lowerCAmelCase     )

      # Ensure is_fast is correctly restored
      self.assertEqual(be_restored.is_fast		,					be_original.is_fast     )

      # Ensure encodings are potentially correctly restored
      if be_original.is_fast:
       self.assertIsNotNone(be_restored.encodings     )
      else:
       self.assertIsNone(be_restored.encodings     )

      # Ensure the keys are the same
      for original_v, restored_v in zip(be_original.values()		,					be_restored.values()     ):
       if equal_op:
        self.assertTrue(equal_op(__lowerCAmelCase		,					__lowerCAmelCase     )     )
       else:
        self.assertEqual(__lowerCAmelCase		,					__lowerCAmelCase     )




     @slow
     def 	__snake_case				(      self					:				List[Any]     )			->    Tuple:
      self.check_tokenizer_from_pretrained(__lowerCAmelCase     )




     def 	__snake_case				(      self					:				str     )			->    Any:
      self.assertEqual(TensorType('tf'     )		,					TensorType.TENSORFLOW     )
      self.assertEqual(TensorType('pt'     )		,					TensorType.PYTORCH     )
      self.assertEqual(TensorType('np'     )		,					TensorType.NUMPY     )




     @require_tokenizers
     def 	__snake_case				(      self					:				int     )			->    Optional[Any]:
      import numpy as np

      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )
      lowerCamelCase__ 					=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

      # Python no tensor
      with self.subTest('BatchEncoding (Python, return_tensors=None)'     ):
       self.assert_dump_and_restore(tokenizer_p('Small example to encode'     )     )

      with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'     ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'		,					return_tensors=TensorType.NUMPY     )		,					np.array_equal     )

      with self.subTest('BatchEncoding (Rust, return_tensors=None)'     ):
       self.assert_dump_and_restore(tokenizer_r('Small example to encode'     )     )

      with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'     ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'		,					return_tensors=TensorType.NUMPY     )		,					np.array_equal     )




     @require_tf
     @require_tokenizers
     def 	__snake_case				(      self					:				Any     )			->    Tuple:
      import tensorflow as tf

      def tf_array_equals(A__					:				Tuple		,					A__					:				Dict     ):
       return tf.reduce_all(tf.equal(__lowerCAmelCase		,					__lowerCAmelCase     )     )

      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )
      lowerCamelCase__ 					=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

      with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'     ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'		,					return_tensors=TensorType.TENSORFLOW     )		,					__lowerCAmelCase     )

      with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'     ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'		,					return_tensors=TensorType.TENSORFLOW     )		,					__lowerCAmelCase     )




     @require_torch
     @require_tokenizers
     def 	__snake_case				(      self					:				Union[str, Any]     )			->    int:
      import torch

      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )
      lowerCamelCase__ 					=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

      with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'     ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'		,					return_tensors=TensorType.PYTORCH     )		,					torch.equal     )

      with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'     ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'		,					return_tensors=TensorType.PYTORCH     )		,					torch.equal     )




     @require_tokenizers
     def 	__snake_case				(      self					:				Any     )			->    Optional[Any]:
      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )
      lowerCamelCase__ 					=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

      with self.subTest('Python Tokenizer'     ):
       self.assertFalse(tokenizer_p('Small example to_encode'     ).is_fast     )

      with self.subTest('Rust Tokenizer'     ):
       self.assertTrue(tokenizer_r('Small example to_encode'     ).is_fast     )




     @require_tokenizers
     def 	__snake_case				(      self					:				List[str]     )			->    Optional[Any]:
      lowerCamelCase__ 					=			BertTokenizerFast.from_pretrained('bert-base-cased'     )
      lowerCamelCase__ 					=			tokenizer_r(['Test', '\xad', 'test']		,					is_split_into_words=__lowerCAmelCase     )

      self.assertEqual(encoded.word_to_tokens(0     )		,					TokenSpan(start=1		,					end=2     )     )
      self.assertEqual(encoded.word_to_tokens(1     )		,					__lowerCAmelCase     )
      self.assertEqual(encoded.word_to_tokens(2     )		,					TokenSpan(start=2		,					end=3     )     )




     def 	__snake_case				(      self					:				int     )			->    str:
      lowerCamelCase__ 					=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='np'     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(2, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(2,)     )
      # test converting the converted
      with CaptureStderr() as cs:
       lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='np'     )
      self.assertFalse(len(cs.err     )		,					msg=F"""should have no warning, but got {cs.err}"""     )

      lowerCamelCase__ 					=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='np'		,					prepend_batch_axis=__lowerCAmelCase     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(1, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(1,)     )




     @require_torch
     def 	__snake_case				(      self					:				int     )			->    Union[str, Any]:
      lowerCamelCase__ 					=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='pt'     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(2, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(2,)     )
      # test converting the converted
      with CaptureStderr() as cs:
       lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='pt'     )
      self.assertFalse(len(cs.err     )		,					msg=F"""should have no warning, but got {cs.err}"""     )

      lowerCamelCase__ 					=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='pt'		,					prepend_batch_axis=__lowerCAmelCase     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(1, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(1,)     )




     @require_tf
     def 	__snake_case				(      self					:				str     )			->    List[str]:
      lowerCamelCase__ 					=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='tf'     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(2, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(2,)     )
      # test converting the converted
      with CaptureStderr() as cs:
       lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='tf'     )
      self.assertFalse(len(cs.err     )		,					msg=F"""should have no warning, but got {cs.err}"""     )

      lowerCamelCase__ 					=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='tf'		,					prepend_batch_axis=__lowerCAmelCase     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(1, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(1,)     )




     @require_flax
     def 	__snake_case				(      self					:				Optional[int]     )			->    str:
      lowerCamelCase__ 					=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='jax'     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(2, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(2,)     )
      # test converting the converted
      with CaptureStderr() as cs:
       lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='jax'     )
      self.assertFalse(len(cs.err     )		,					msg=F"""should have no warning, but got {cs.err}"""     )

      lowerCamelCase__ 					=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
      lowerCamelCase__ 					=			batch.convert_to_tensors(tensor_type='jax'		,					prepend_batch_axis=__lowerCAmelCase     )
      self.assertEqual(tensor_batch['inputs'].shape		,					(1, 3)     )
      self.assertEqual(tensor_batch['labels'].shape		,					(1,)     )




     def 	__snake_case				(      self					:				Union[str, Any]     )			->    List[Any]:
      lowerCamelCase__ 					=			[{'input_ids': np.array([0, 1, 2]     )}, {'input_ids': np.array([0, 1, 2, 3]     )}]
      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )

      lowerCamelCase__ 					=			tokenizer.pad(__lowerCAmelCase		,					padding=__lowerCAmelCase     )
      self.assertTrue(isinstance(batch['input_ids']		,					np.ndarray     )     )
      self.assertEqual(batch['input_ids'].tolist()		,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
      lowerCamelCase__ 					=			tokenizer.pad(__lowerCAmelCase		,					padding=__lowerCAmelCase		,					return_tensors='np'     )
      self.assertTrue(isinstance(batch['input_ids']		,					np.ndarray     )     )
      self.assertEqual(batch['input_ids'].tolist()		,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )




     @require_torch
     def 	__snake_case				(      self					:				Optional[int]     )			->    Dict:
      import torch

      lowerCamelCase__ 					=			[{'input_ids': torch.tensor([0, 1, 2]     )}, {'input_ids': torch.tensor([0, 1, 2, 3]     )}]
      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )

      lowerCamelCase__ 					=			tokenizer.pad(__lowerCAmelCase		,					padding=__lowerCAmelCase     )
      self.assertTrue(isinstance(batch['input_ids']		,					torch.Tensor     )     )
      self.assertEqual(batch['input_ids'].tolist()		,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
      lowerCamelCase__ 					=			tokenizer.pad(__lowerCAmelCase		,					padding=__lowerCAmelCase		,					return_tensors='pt'     )
      self.assertTrue(isinstance(batch['input_ids']		,					torch.Tensor     )     )
      self.assertEqual(batch['input_ids'].tolist()		,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )




     @require_tf
     def 	__snake_case				(      self					:				Tuple     )			->    Optional[int]:
      import tensorflow as tf

      lowerCamelCase__ 					=			[{'input_ids': tf.constant([0, 1, 2]     )}, {'input_ids': tf.constant([0, 1, 2, 3]     )}]
      lowerCamelCase__ 					=			BertTokenizer.from_pretrained('bert-base-cased'     )

      lowerCamelCase__ 					=			tokenizer.pad(__lowerCAmelCase		,					padding=__lowerCAmelCase     )
      self.assertTrue(isinstance(batch['input_ids']		,					tf.Tensor     )     )
      self.assertEqual(batch['input_ids'].numpy().tolist()		,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
      lowerCamelCase__ 					=			tokenizer.pad(__lowerCAmelCase		,					padding=__lowerCAmelCase		,					return_tensors='tf'     )
      self.assertTrue(isinstance(batch['input_ids']		,					tf.Tensor     )     )
      self.assertEqual(batch['input_ids'].numpy().tolist()		,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )




     @require_tokenizers
     def 	__snake_case				(      self					:				Optional[int]     )			->    List[Any]:
      lowerCamelCase__ 					=			Tokenizer(WordPiece(unk_token='[UNK]'     )     )
      PreTrainedTokenizerFast(tokenizer_object=__lowerCAmelCase     )




     @require_tokenizers
     def 	__snake_case				(      self					:				int     )			->    List[Any]:
      lowerCamelCase__ 					=			Tokenizer(WordPiece(unk_token='[UNK]'     )     )
      with tempfile.TemporaryDirectory() as tmpdirname:
       bert_tokenizer.save(os.path.join(__lowerCAmelCase		,					'tokenizer.json'     )     )
       PreTrainedTokenizerFast(tokenizer_file=os.path.join(__lowerCAmelCase		,					'tokenizer.json'     )     )





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__SCREAMING_SNAKE_CASE      :				int				      ={"""configuration_nat""": ["""NAT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """NatConfig"""]}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __SCREAMING_SNAKE_CASE      :				Optional[int]				      =[
          """NAT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """NatForImageClassification""",
          """NatModel""",
          """NatPreTrainedModel""",
          """NatBackbone""",
      ]

if TYPE_CHECKING:
      from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_nat import (
                NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
                NatBackbone,
                NatForImageClassification,
                NatModel,
                NatPreTrainedModel,
            )

else:
      import sys

      __SCREAMING_SNAKE_CASE      :				Any				      =_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)