

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
  from tokenizers import Tokenizer
  from tokenizers.models import WordPiece






class 							A_			(		unittest.TestCase ):


     def   UpperCAmelCase  (							self      :		List[Any]      ,  UpperCAmelCase      :		int )							->     List[Any]:
      __lowerCAmelCase:       Dict            =				list(tokenizer_class.max_model_input_sizes.keys() )
      for model_name in sa_models[:1]:
       __lowerCAmelCase:       Any            =				tokenizer_class.from_pretrained(__lowerCAmelCase )
       self.assertIsNotNone(__lowerCAmelCase )
       self.assertIsInstance(__lowerCAmelCase      ,  __lowerCAmelCase )
       self.assertIsInstance(__lowerCAmelCase      ,  __lowerCAmelCase )

       for special_tok in tokenizer.all_special_tokens:
        self.assertIsInstance(__lowerCAmelCase      ,  __lowerCAmelCase )
        __lowerCAmelCase:       int            =				tokenizer.convert_tokens_to_ids(__lowerCAmelCase )
        self.assertIsInstance(__lowerCAmelCase      ,  __lowerCAmelCase )


     def   UpperCAmelCase  (							self      :		List[str]      ,  UpperCAmelCase      :		str      ,  UpperCAmelCase      :		Tuple = None )							->     Optional[Any]:
      __lowerCAmelCase:       List[Any]            =				pickle.dumps(__lowerCAmelCase )
      self.assertIsNotNone(__lowerCAmelCase )

      __lowerCAmelCase:       Any            =				pickle.loads(__lowerCAmelCase )

      # Ensure is_fast is correctly restored
      self.assertEqual(be_restored.is_fast      ,  be_original.is_fast )

      # Ensure encodings are potentially correctly restored
      if be_original.is_fast:
       self.assertIsNotNone(be_restored.encodings )
      else:
       self.assertIsNone(be_restored.encodings )

      # Ensure the keys are the same
      for original_v, restored_v in zip(be_original.values()      ,  be_restored.values() ):
       if equal_op:
        self.assertTrue(equal_op(__lowerCAmelCase      ,  __lowerCAmelCase ) )
       else:
        self.assertEqual(__lowerCAmelCase      ,  __lowerCAmelCase )


     @slow
     def   UpperCAmelCase  (							self      :		Optional[Any] )							->     Tuple:
      self.check_tokenizer_from_pretrained(__lowerCAmelCase )


     def   UpperCAmelCase  (							self      :		Dict )							->     Dict:
      self.assertEqual(TensorType('tf' )      ,  TensorType.TENSORFLOW )
      self.assertEqual(TensorType('pt' )      ,  TensorType.PYTORCH )
      self.assertEqual(TensorType('np' )      ,  TensorType.NUMPY )


     @require_tokenizers
     def   UpperCAmelCase  (							self      :		Any )							->     Optional[int]:
      import numpy as np

      __lowerCAmelCase:       List[str]            =				BertTokenizer.from_pretrained('bert-base-cased' )
      __lowerCAmelCase:       Any            =				BertTokenizerFast.from_pretrained('bert-base-cased' )

      # Python no tensor
      with self.subTest('BatchEncoding (Python, return_tensors=None)' ):
       self.assert_dump_and_restore(tokenizer_p('Small example to encode' ) )

      with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)' ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'      ,  return_tensors=TensorType.NUMPY )      ,  np.array_equal )

      with self.subTest('BatchEncoding (Rust, return_tensors=None)' ):
       self.assert_dump_and_restore(tokenizer_r('Small example to encode' ) )

      with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)' ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'      ,  return_tensors=TensorType.NUMPY )      ,  np.array_equal )


     @require_tf
     @require_tokenizers
     def   UpperCAmelCase  (							self      :		Dict )							->     Any:
      import tensorflow as tf

      def tf_array_equals(UpperCAmelCase      :		Optional[int]      ,  UpperCAmelCase      :		Dict ):
       return tf.reduce_all(tf.equal(__lowerCAmelCase      ,  __lowerCAmelCase ) )

      __lowerCAmelCase:       Optional[int]            =				BertTokenizer.from_pretrained('bert-base-cased' )
      __lowerCAmelCase:       List[Any]            =				BertTokenizerFast.from_pretrained('bert-base-cased' )

      with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)' ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'      ,  return_tensors=TensorType.TENSORFLOW )      ,  __lowerCAmelCase )

      with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)' ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'      ,  return_tensors=TensorType.TENSORFLOW )      ,  __lowerCAmelCase )


     @require_torch
     @require_tokenizers
     def   UpperCAmelCase  (							self      :		Optional[int] )							->     Union[str, Any]:
      import torch

      __lowerCAmelCase:       List[Any]            =				BertTokenizer.from_pretrained('bert-base-cased' )
      __lowerCAmelCase:       Optional[Any]            =				BertTokenizerFast.from_pretrained('bert-base-cased' )

      with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)' ):
       self.assert_dump_and_restore(
           tokenizer_p('Small example to encode'      ,  return_tensors=TensorType.PYTORCH )      ,  torch.equal )

      with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)' ):
       self.assert_dump_and_restore(
           tokenizer_r('Small example to encode'      ,  return_tensors=TensorType.PYTORCH )      ,  torch.equal )


     @require_tokenizers
     def   UpperCAmelCase  (							self      :		Tuple )							->     Tuple:
      __lowerCAmelCase:       Optional[int]            =				BertTokenizer.from_pretrained('bert-base-cased' )
      __lowerCAmelCase:       Optional[Any]            =				BertTokenizerFast.from_pretrained('bert-base-cased' )

      with self.subTest('Python Tokenizer' ):
       self.assertFalse(tokenizer_p('Small example to_encode' ).is_fast )

      with self.subTest('Rust Tokenizer' ):
       self.assertTrue(tokenizer_r('Small example to_encode' ).is_fast )


     @require_tokenizers
     def   UpperCAmelCase  (							self      :		Dict )							->     Optional[int]:
      __lowerCAmelCase:       List[str]            =				BertTokenizerFast.from_pretrained('bert-base-cased' )
      __lowerCAmelCase:       Tuple            =				tokenizer_r(['Test', '\xad', 'test']      ,  is_split_into_words=__lowerCAmelCase )

      self.assertEqual(encoded.word_to_tokens(0 )      ,  TokenSpan(start=1      ,  end=2 ) )
      self.assertEqual(encoded.word_to_tokens(1 )      ,  __lowerCAmelCase )
      self.assertEqual(encoded.word_to_tokens(2 )      ,  TokenSpan(start=2      ,  end=3 ) )


     def   UpperCAmelCase  (							self      :		Union[str, Any] )							->     str:
      __lowerCAmelCase:       Any            =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
      __lowerCAmelCase:       Optional[int]            =				batch.convert_to_tensors(tensor_type='np' )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (2, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (2,) )
      # test converting the converted
      with CaptureStderr() as cs:
       __lowerCAmelCase:       Dict            =				batch.convert_to_tensors(tensor_type='np' )
      self.assertFalse(len(cs.err )      ,  msg=F'''should have no warning, but got {cs.err}''' )

      __lowerCAmelCase:       List[str]            =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
      __lowerCAmelCase:       str            =				batch.convert_to_tensors(tensor_type='np'      ,  prepend_batch_axis=__lowerCAmelCase )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (1, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (1,) )


     @require_torch
     def   UpperCAmelCase  (							self      :		Any )							->     Optional[int]:
      __lowerCAmelCase:       str            =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
      __lowerCAmelCase:       Optional[Any]            =				batch.convert_to_tensors(tensor_type='pt' )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (2, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (2,) )
      # test converting the converted
      with CaptureStderr() as cs:
       __lowerCAmelCase:       Optional[Any]            =				batch.convert_to_tensors(tensor_type='pt' )
      self.assertFalse(len(cs.err )      ,  msg=F'''should have no warning, but got {cs.err}''' )

      __lowerCAmelCase:       List[Any]            =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
      __lowerCAmelCase:       str            =				batch.convert_to_tensors(tensor_type='pt'      ,  prepend_batch_axis=__lowerCAmelCase )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (1, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (1,) )


     @require_tf
     def   UpperCAmelCase  (							self      :		Union[str, Any] )							->     List[str]:
      __lowerCAmelCase:       Dict            =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
      __lowerCAmelCase:       Optional[Any]            =				batch.convert_to_tensors(tensor_type='tf' )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (2, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (2,) )
      # test converting the converted
      with CaptureStderr() as cs:
       __lowerCAmelCase:       Union[str, Any]            =				batch.convert_to_tensors(tensor_type='tf' )
      self.assertFalse(len(cs.err )      ,  msg=F'''should have no warning, but got {cs.err}''' )

      __lowerCAmelCase:       List[str]            =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
      __lowerCAmelCase:       Optional[Any]            =				batch.convert_to_tensors(tensor_type='tf'      ,  prepend_batch_axis=__lowerCAmelCase )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (1, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (1,) )


     @require_flax
     def   UpperCAmelCase  (							self      :		str )							->     Dict:
      __lowerCAmelCase:       Tuple            =				BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]} )
      __lowerCAmelCase:       Optional[Any]            =				batch.convert_to_tensors(tensor_type='jax' )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (2, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (2,) )
      # test converting the converted
      with CaptureStderr() as cs:
       __lowerCAmelCase:       Optional[Any]            =				batch.convert_to_tensors(tensor_type='jax' )
      self.assertFalse(len(cs.err )      ,  msg=F'''should have no warning, but got {cs.err}''' )

      __lowerCAmelCase:       Dict            =				BatchEncoding({'inputs': [1, 2, 3], 'labels': 0} )
      __lowerCAmelCase:       int            =				batch.convert_to_tensors(tensor_type='jax'      ,  prepend_batch_axis=__lowerCAmelCase )
      self.assertEqual(tensor_batch['inputs'].shape      ,  (1, 3) )
      self.assertEqual(tensor_batch['labels'].shape      ,  (1,) )


     def   UpperCAmelCase  (							self      :		Optional[Any] )							->     Union[str, Any]:
      __lowerCAmelCase:       Dict            =				[{'input_ids': np.array([0, 1, 2] )}, {'input_ids': np.array([0, 1, 2, 3] )}]
      __lowerCAmelCase:       int            =				BertTokenizer.from_pretrained('bert-base-cased' )

      __lowerCAmelCase:       int            =				tokenizer.pad(__lowerCAmelCase      ,  padding=__lowerCAmelCase )
      self.assertTrue(isinstance(batch['input_ids']      ,  np.ndarray ) )
      self.assertEqual(batch['input_ids'].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
      __lowerCAmelCase:       int            =				tokenizer.pad(__lowerCAmelCase      ,  padding=__lowerCAmelCase      ,  return_tensors='np' )
      self.assertTrue(isinstance(batch['input_ids']      ,  np.ndarray ) )
      self.assertEqual(batch['input_ids'].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )


     @require_torch
     def   UpperCAmelCase  (							self      :		Optional[Any] )							->     Tuple:
      import torch

      __lowerCAmelCase:       Optional[int]            =				[{'input_ids': torch.tensor([0, 1, 2] )}, {'input_ids': torch.tensor([0, 1, 2, 3] )}]
      __lowerCAmelCase:       str            =				BertTokenizer.from_pretrained('bert-base-cased' )

      __lowerCAmelCase:       str            =				tokenizer.pad(__lowerCAmelCase      ,  padding=__lowerCAmelCase )
      self.assertTrue(isinstance(batch['input_ids']      ,  torch.Tensor ) )
      self.assertEqual(batch['input_ids'].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
      __lowerCAmelCase:       Tuple            =				tokenizer.pad(__lowerCAmelCase      ,  padding=__lowerCAmelCase      ,  return_tensors='pt' )
      self.assertTrue(isinstance(batch['input_ids']      ,  torch.Tensor ) )
      self.assertEqual(batch['input_ids'].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )


     @require_tf
     def   UpperCAmelCase  (							self      :		Union[str, Any] )							->     Any:
      import tensorflow as tf

      __lowerCAmelCase:       int            =				[{'input_ids': tf.constant([0, 1, 2] )}, {'input_ids': tf.constant([0, 1, 2, 3] )}]
      __lowerCAmelCase:       Dict            =				BertTokenizer.from_pretrained('bert-base-cased' )

      __lowerCAmelCase:       Optional[Any]            =				tokenizer.pad(__lowerCAmelCase      ,  padding=__lowerCAmelCase )
      self.assertTrue(isinstance(batch['input_ids']      ,  tf.Tensor ) )
      self.assertEqual(batch['input_ids'].numpy().tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
      __lowerCAmelCase:       Union[str, Any]            =				tokenizer.pad(__lowerCAmelCase      ,  padding=__lowerCAmelCase      ,  return_tensors='tf' )
      self.assertTrue(isinstance(batch['input_ids']      ,  tf.Tensor ) )
      self.assertEqual(batch['input_ids'].numpy().tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )


     @require_tokenizers
     def   UpperCAmelCase  (							self      :		Optional[int] )							->     int:
      __lowerCAmelCase:       Any            =				Tokenizer(WordPiece(unk_token='[UNK]' ) )
      PreTrainedTokenizerFast(tokenizer_object=__lowerCAmelCase )


     @require_tokenizers
     def   UpperCAmelCase  (							self      :		int )							->     List[str]:
      __lowerCAmelCase:       int            =				Tokenizer(WordPiece(unk_token='[UNK]' ) )
      with tempfile.TemporaryDirectory() as tmpdirname:
       bert_tokenizer.save(os.path.join(__lowerCAmelCase      ,  'tokenizer.json' ) )
       PreTrainedTokenizerFast(tokenizer_file=os.path.join(__lowerCAmelCase      ,  'tokenizer.json' ) )








from __future__ import annotations

from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any






class 							A_			:
     _lowercase      :					Any       =     ('_front', '_back', '_len')






     @dataclass
     class 							A_			:
          _lowercase      :					Any       =     None
          _lowercase      :					Deque._Node | None       =     None
          _lowercase      :					Deque._Node | None       =     None







     class 							A_			:
          _lowercase      :					Optional[Any]       =     ('_cur',)


          def __init__(							self      :		Any      ,  UpperCAmelCase      :		Deque._Node | None )							->     None:
           __lowerCAmelCase:       int            =				cur


          def __iter__(							self      :		int )							->     Deque._Iterator:
           return self


          def   UpperCAmelCase  (							self      :		Optional[Any] )							->     Any:
           if self._cur is None:
            # finished iterating
            raise StopIteration
           __lowerCAmelCase:       Optional[Any]            =				self._cur.val
           __lowerCAmelCase:       List[Any]            =				self._cur.next_node

           return val







     def __init__(							self      :		Any      ,  UpperCAmelCase      :		Iterable[Any] | None = None )							->     None:
      __lowerCAmelCase:       Any            =				None
      __lowerCAmelCase:       Any            =				None
      __lowerCAmelCase:       int            =				0

      if iterable is not None:
       # append every value to the deque
       for val in iterable:
        self.append(UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		List[str]      ,  UpperCAmelCase      :		Any )							->     None:
      __lowerCAmelCase:       Union[str, Any]            =				self._Node(UpperCAmelCase      ,  UpperCAmelCase      ,  UpperCAmelCase )
      if self.is_empty():
       # front = back
       __lowerCAmelCase:       int            =				node
       __lowerCAmelCase:       List[Any]            =				1
      else:
       # connect nodes
       __lowerCAmelCase:       Optional[Any]            =				node
       __lowerCAmelCase:       str            =				self._back
       __lowerCAmelCase:       Optional[int]            =				node  # assign new back to the new node

       self._len += 1

       # make sure there were no errors
       assert not self.is_empty(), "Error on appending value."


     def   UpperCAmelCase  (							self      :		Optional[int]      ,  UpperCAmelCase      :		Any )							->     None:
      __lowerCAmelCase:       int            =				self._Node(UpperCAmelCase      ,  UpperCAmelCase      ,  UpperCAmelCase )
      if self.is_empty():
       # front = back
       __lowerCAmelCase:       Union[str, Any]            =				node
       __lowerCAmelCase:       str            =				1
      else:
       # connect nodes
       __lowerCAmelCase:       Dict            =				self._front
       __lowerCAmelCase:       Optional[int]            =				node
       __lowerCAmelCase:       Any            =				node  # assign new front to the new node

       self._len += 1

       # make sure there were no errors
       assert not self.is_empty(), "Error on appending value."


     def   UpperCAmelCase  (							self      :		Union[str, Any]      ,  UpperCAmelCase      :		Iterable[Any] )							->     None:
      for val in iterable:
       self.append(UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		List[str]      ,  UpperCAmelCase      :		Iterable[Any] )							->     None:
      for val in iterable:
       self.appendleft(UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		Dict )							->     Any:
      assert not self.is_empty(), "Deque is empty."

      __lowerCAmelCase:       Optional[int]            =				self._back
      __lowerCAmelCase:       Optional[Any]            =				self._back.prev_node  # set new back
      # drop the last node - python will deallocate memory automatically
      __lowerCAmelCase:       Optional[int]            =				None

      self._len -= 1

      return topop.val


     def   UpperCAmelCase  (							self      :		Dict )							->     Any:
      assert not self.is_empty(), "Deque is empty."

      __lowerCAmelCase:       Optional[Any]            =				self._front
      __lowerCAmelCase:       Optional[Any]            =				self._front.next_node  # set new front and drop the first node
      __lowerCAmelCase:       int            =				None

      self._len -= 1

      return topop.val


     def   UpperCAmelCase  (							self      :		Dict )							->     bool:
      return self._front is None


     def __len__(							self      :		List[Any] )							->     int:
      return self._len


     def __eq__(							self      :		Union[str, Any]      ,  UpperCAmelCase      :		object )							->     bool:

      if not isinstance(UpperCAmelCase      ,  UpperCAmelCase ):
       return NotImplemented

      __lowerCAmelCase:       List[Any]            =				self._front
      __lowerCAmelCase:       Dict            =				other._front

      # if the length of the dequeues are not the same, they are not equal
      if len(self ) != len(UpperCAmelCase ):
       return False

      while me is not None and oth is not None:
       # compare every value
       if me.val != oth.val:
        return False
       __lowerCAmelCase:       Optional[Any]            =				me.next_node
       __lowerCAmelCase:       Optional[Any]            =				oth.next_node

      return True


     def __iter__(							self      :		Tuple )							->     Deque._Iterator:
      return Deque._Iterator(self._front )


     def __repr__(							self      :		Optional[Any] )							->     str:
      __lowerCAmelCase:       Union[str, Any]            =				[]
      __lowerCAmelCase:       Optional[int]            =				self._front
      while aux is not None:
       # append the values in a list to display
       values_list.append(aux.val )
       __lowerCAmelCase:       Any            =				aux.next_node

      return F'''[{', '.join(repr(UpperCAmelCase ) for val in values_list )}]'''


if __name__ == "__main__":
  import doctest

  doctest.testmod()





