
import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
 from tokenizers import Tokenizer
 from tokenizers.models import WordPiece


class   __lowerCAmelCase		(		unittest.TestCase ):



      """simple docstring"""







      def 						snake_case_     ( self						:    Union[str, Any]			,    _snake_case						:    Dict			):
          __lowercase						:     Optional[Any]		      =    list(tokenizer_class.max_model_input_sizes.keys()			)
          for model_name in sa_models[:1]:
              __lowercase						:     Tuple		      =    tokenizer_class.from_pretrained(_snake_case			)
              self.assertIsNotNone(_snake_case			)
              self.assertIsInstance(_snake_case			,    _snake_case			)
              self.assertIsInstance(_snake_case			,    _snake_case			)

              for special_tok in tokenizer.all_special_tokens:
                  self.assertIsInstance(_snake_case			,    _snake_case			)
                  __lowercase						:     List[Any]		      =    tokenizer.convert_tokens_to_ids(_snake_case			)
                  self.assertIsInstance(_snake_case			,    _snake_case			)




      def 						snake_case_     ( self						:    Optional[Any]			,    _snake_case						:    BatchEncoding			,    _snake_case						:    Optional[Callable] = None			):
          __lowercase						:     List[str]		      =    pickle.dumps(_snake_case			)
          self.assertIsNotNone(_snake_case			)

          __lowercase						:     int		      =    pickle.loads(_snake_case			)

          # Ensure is_fast is correctly restored
          self.assertEqual(be_restored.is_fast			,    be_original.is_fast			)

          # Ensure encodings are potentially correctly restored
          if be_original.is_fast:
              self.assertIsNotNone(be_restored.encodings			)
          else:
              self.assertIsNone(be_restored.encodings			)

          # Ensure the keys are the same
          for original_v, restored_v in zip(be_original.values()			,    be_restored.values()			):
              if equal_op:
                  self.assertTrue(equal_op(_snake_case			,    _snake_case			)			)
              else:
                  self.assertEqual(_snake_case			,    _snake_case			)




      @slow
      def 						snake_case_     ( self						:    Optional[Any]			):
          self.check_tokenizer_from_pretrained(_snake_case			)




      def 						snake_case_     ( self						:    Optional[Any]			):
          self.assertEqual(TensorType('''tf'''			)			,    TensorType.TENSORFLOW			)
          self.assertEqual(TensorType('''pt'''			)			,    TensorType.PYTORCH			)
          self.assertEqual(TensorType('''np'''			)			,    TensorType.NUMPY			)




      @require_tokenizers
      def 						snake_case_     ( self						:    Optional[Any]			):
          import numpy as np

          __lowercase						:     int		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)
          __lowercase						:     List[str]		      =    BertTokenizerFast.from_pretrained('''bert-base-cased'''			)

          # Python no tensor
          with self.subTest('''BatchEncoding (Python, return_tensors=None)'''			):
              self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''			)			)

          with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''			):
              self.assert_dump_and_restore(
                  tokenizer_p('''Small example to encode'''			,    return_tensors=TensorType.NUMPY			)			,    np.array_equal			)

          with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''			):
              self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''			)			)

          with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''			):
              self.assert_dump_and_restore(
                  tokenizer_r('''Small example to encode'''			,    return_tensors=TensorType.NUMPY			)			,    np.array_equal			)




      @require_tf
      @require_tokenizers
      def 						snake_case_     ( self						:    Optional[Any]			):
          import tensorflow as tf

          def tf_array_equals(_snake_case						:    Union[str, Any]			,    _snake_case						:    Union[str, Any]			):
              return tf.reduce_all(tf.equal(_snake_case			,    _snake_case			)			)

          __lowercase						:     List[str]		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)
          __lowercase						:     int		      =    BertTokenizerFast.from_pretrained('''bert-base-cased'''			)

          with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''			):
              self.assert_dump_and_restore(
                  tokenizer_p('''Small example to encode'''			,    return_tensors=TensorType.TENSORFLOW			)			,    _snake_case			)

          with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''			):
              self.assert_dump_and_restore(
                  tokenizer_r('''Small example to encode'''			,    return_tensors=TensorType.TENSORFLOW			)			,    _snake_case			)




      @require_torch
      @require_tokenizers
      def 						snake_case_     ( self						:    Union[str, Any]			):
          import torch

          __lowercase						:     Tuple		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)
          __lowercase						:     List[Any]		      =    BertTokenizerFast.from_pretrained('''bert-base-cased'''			)

          with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''			):
              self.assert_dump_and_restore(
                  tokenizer_p('''Small example to encode'''			,    return_tensors=TensorType.PYTORCH			)			,    torch.equal			)

          with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''			):
              self.assert_dump_and_restore(
                  tokenizer_r('''Small example to encode'''			,    return_tensors=TensorType.PYTORCH			)			,    torch.equal			)




      @require_tokenizers
      def 						snake_case_     ( self						:    str			):
          __lowercase						:     Optional[int]		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)
          __lowercase						:     Union[str, Any]		      =    BertTokenizerFast.from_pretrained('''bert-base-cased'''			)

          with self.subTest('''Python Tokenizer'''			):
              self.assertFalse(tokenizer_p('''Small example to_encode'''			).is_fast			)

          with self.subTest('''Rust Tokenizer'''			):
              self.assertTrue(tokenizer_r('''Small example to_encode'''			).is_fast			)




      @require_tokenizers
      def 						snake_case_     ( self						:    List[str]			):
          __lowercase						:     Dict		      =    BertTokenizerFast.from_pretrained('''bert-base-cased'''			)
          __lowercase						:     List[Any]		      =    tokenizer_r(['''Test''', '''\xad''', '''test''']			,    is_split_into_words=_snake_case			)

          self.assertEqual(encoded.word_to_tokens(0			)			,    TokenSpan(start=1			,    end=2			)			)
          self.assertEqual(encoded.word_to_tokens(1			)			,    _snake_case			)
          self.assertEqual(encoded.word_to_tokens(2			)			,    TokenSpan(start=2			,    end=3			)			)




      def 						snake_case_     ( self						:    Optional[Any]			):
          __lowercase						:     int		      =    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}			)
          __lowercase						:     Any		      =    batch.convert_to_tensors(tensor_type='''np'''			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (2, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (2,)			)
          # test converting the converted
          with CaptureStderr() as cs:
              __lowercase						:     Optional[int]		      =    batch.convert_to_tensors(tensor_type='''np'''			)
          self.assertFalse(len(cs.err			)			,    msg=F'should have no warning, but got {cs.err}'			)

          __lowercase						:     Optional[Any]		      =    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}			)
          __lowercase						:     Union[str, Any]		      =    batch.convert_to_tensors(tensor_type='''np'''			,    prepend_batch_axis=_snake_case			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (1, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (1,)			)




      @require_torch
      def 						snake_case_     ( self						:    Optional[int]			):
          __lowercase						:     str		      =    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}			)
          __lowercase						:     List[Any]		      =    batch.convert_to_tensors(tensor_type='''pt'''			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (2, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (2,)			)
          # test converting the converted
          with CaptureStderr() as cs:
              __lowercase						:     Optional[int]		      =    batch.convert_to_tensors(tensor_type='''pt'''			)
          self.assertFalse(len(cs.err			)			,    msg=F'should have no warning, but got {cs.err}'			)

          __lowercase						:     Any		      =    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}			)
          __lowercase						:     List[str]		      =    batch.convert_to_tensors(tensor_type='''pt'''			,    prepend_batch_axis=_snake_case			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (1, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (1,)			)




      @require_tf
      def 						snake_case_     ( self						:    str			):
          __lowercase						:     Any		      =    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}			)
          __lowercase						:     List[Any]		      =    batch.convert_to_tensors(tensor_type='''tf'''			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (2, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (2,)			)
          # test converting the converted
          with CaptureStderr() as cs:
              __lowercase						:     Union[str, Any]		      =    batch.convert_to_tensors(tensor_type='''tf'''			)
          self.assertFalse(len(cs.err			)			,    msg=F'should have no warning, but got {cs.err}'			)

          __lowercase						:     Tuple		      =    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}			)
          __lowercase						:     List[str]		      =    batch.convert_to_tensors(tensor_type='''tf'''			,    prepend_batch_axis=_snake_case			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (1, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (1,)			)




      @require_flax
      def 						snake_case_     ( self						:    Optional[int]			):
          __lowercase						:     Optional[Any]		      =    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}			)
          __lowercase						:     Optional[int]		      =    batch.convert_to_tensors(tensor_type='''jax'''			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (2, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (2,)			)
          # test converting the converted
          with CaptureStderr() as cs:
              __lowercase						:     Tuple		      =    batch.convert_to_tensors(tensor_type='''jax'''			)
          self.assertFalse(len(cs.err			)			,    msg=F'should have no warning, but got {cs.err}'			)

          __lowercase						:     Any		      =    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}			)
          __lowercase						:     int		      =    batch.convert_to_tensors(tensor_type='''jax'''			,    prepend_batch_axis=_snake_case			)
          self.assertEqual(tensor_batch['''inputs'''].shape			,    (1, 3)			)
          self.assertEqual(tensor_batch['''labels'''].shape			,    (1,)			)




      def 						snake_case_     ( self						:    List[Any]			):
          __lowercase						:     Dict		      =    [{'''input_ids''': np.array([0, 1, 2]			)}, {'''input_ids''': np.array([0, 1, 2, 3]			)}]
          __lowercase						:     List[Any]		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)

          __lowercase						:     Union[str, Any]		      =    tokenizer.pad(_snake_case			,    padding=_snake_case			)
          self.assertTrue(isinstance(batch['''input_ids''']			,    np.ndarray			)			)
          self.assertEqual(batch['''input_ids'''].tolist()			,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
          __lowercase						:     List[str]		      =    tokenizer.pad(_snake_case			,    padding=_snake_case			,    return_tensors='''np'''			)
          self.assertTrue(isinstance(batch['''input_ids''']			,    np.ndarray			)			)
          self.assertEqual(batch['''input_ids'''].tolist()			,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)




      @require_torch
      def 						snake_case_     ( self						:    Optional[Any]			):
          import torch

          __lowercase						:     Tuple		      =    [{'''input_ids''': torch.tensor([0, 1, 2]			)}, {'''input_ids''': torch.tensor([0, 1, 2, 3]			)}]
          __lowercase						:     str		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)

          __lowercase						:     Optional[int]		      =    tokenizer.pad(_snake_case			,    padding=_snake_case			)
          self.assertTrue(isinstance(batch['''input_ids''']			,    torch.Tensor			)			)
          self.assertEqual(batch['''input_ids'''].tolist()			,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
          __lowercase						:     Dict		      =    tokenizer.pad(_snake_case			,    padding=_snake_case			,    return_tensors='''pt'''			)
          self.assertTrue(isinstance(batch['''input_ids''']			,    torch.Tensor			)			)
          self.assertEqual(batch['''input_ids'''].tolist()			,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)




      @require_tf
      def 						snake_case_     ( self						:    Optional[Any]			):
          import tensorflow as tf

          __lowercase						:     Any		      =    [{'''input_ids''': tf.constant([0, 1, 2]			)}, {'''input_ids''': tf.constant([0, 1, 2, 3]			)}]
          __lowercase						:     Union[str, Any]		      =    BertTokenizer.from_pretrained('''bert-base-cased'''			)

          __lowercase						:     List[Any]		      =    tokenizer.pad(_snake_case			,    padding=_snake_case			)
          self.assertTrue(isinstance(batch['''input_ids''']			,    tf.Tensor			)			)
          self.assertEqual(batch['''input_ids'''].numpy().tolist()			,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
          __lowercase						:     List[Any]		      =    tokenizer.pad(_snake_case			,    padding=_snake_case			,    return_tensors='''tf'''			)
          self.assertTrue(isinstance(batch['''input_ids''']			,    tf.Tensor			)			)
          self.assertEqual(batch['''input_ids'''].numpy().tolist()			,    [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)




      @require_tokenizers
      def 						snake_case_     ( self						:    Optional[Any]			):
          __lowercase						:     Optional[int]		      =    Tokenizer(WordPiece(unk_token='''[UNK]'''			)			)
          PreTrainedTokenizerFast(tokenizer_object=_snake_case			)




      @require_tokenizers
      def 						snake_case_     ( self						:    Union[str, Any]			):
          __lowercase						:     int		      =    Tokenizer(WordPiece(unk_token='''[UNK]'''			)			)
          with tempfile.TemporaryDirectory() as tmpdirname:
              bert_tokenizer.save(os.path.join(_snake_case			,    '''tokenizer.json'''			)			)
              PreTrainedTokenizerFast(tokenizer_file=os.path.join(_snake_case			,    '''tokenizer.json'''			)			)




__lowerCAmelCase   :     List[Any]      =						"\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

__lowerCAmelCase   :     Optional[Any]      =						[{"type": "code", "content": INSTALL_CONTENT}]
__lowerCAmelCase   :     List[str]      =						{
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}


