

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
       from tokenizers import Tokenizer
       from tokenizers.models import WordPiece
class    A_				( unittest.TestCase  ):


    '''simple docstring'''






    def        lowerCAmelCase_  (self       , lowercase__					)       -> List[str]:
        __UpperCAmelCase			   =     list(tokenizer_class.max_model_input_sizes.keys()					)
        for model_name in sa_models[:1]:
            __UpperCAmelCase			   =     tokenizer_class.from_pretrained(lowercase__					)
            self.assertIsNotNone(lowercase__					)
            self.assertIsInstance(lowercase__       , lowercase__					)
            self.assertIsInstance(lowercase__       , lowercase__					)

            for special_tok in tokenizer.all_special_tokens:
                self.assertIsInstance(lowercase__       , lowercase__					)
                __UpperCAmelCase			   =     tokenizer.convert_tokens_to_ids(lowercase__					)
                self.assertIsInstance(lowercase__       , lowercase__					)


    def        lowerCAmelCase_  (self       , lowercase__       , lowercase__ = None					)       -> Any:
        __UpperCAmelCase			   =     pickle.dumps(lowercase__					)
        self.assertIsNotNone(lowercase__					)

        __UpperCAmelCase			   =     pickle.loads(lowercase__					)

        # Ensure is_fast is correctly restored
        self.assertEqual(be_restored.is_fast       , be_original.is_fast					)

        # Ensure encodings are potentially correctly restored
        if be_original.is_fast:
            self.assertIsNotNone(be_restored.encodings					)
        else:
            self.assertIsNone(be_restored.encodings					)

        # Ensure the keys are the same
        for original_v, restored_v in zip(be_original.values()       , be_restored.values()					):
            if equal_op:
                self.assertTrue(equal_op(lowercase__       , lowercase__					)					)
            else:
                self.assertEqual(lowercase__       , lowercase__					)


    @slow
    def        lowerCAmelCase_  (self					)       -> int:
        self.check_tokenizer_from_pretrained(lowercase__					)


    def        lowerCAmelCase_  (self					)       -> Optional[int]:
        self.assertEqual(TensorType('''tf'''					)       , TensorType.TENSORFLOW					)
        self.assertEqual(TensorType('''pt'''					)       , TensorType.PYTORCH					)
        self.assertEqual(TensorType('''np'''					)       , TensorType.NUMPY					)


    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> Any:
        import numpy as np

        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)
        __UpperCAmelCase			   =     BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

        # Python no tensor
        with self.subTest('''BatchEncoding (Python, return_tensors=None)'''					):
            self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''					)					)

        with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''					):
            self.assert_dump_and_restore(
                tokenizer_p('''Small example to encode'''       , return_tensors=TensorType.NUMPY					)       , np.array_equal					)

        with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''					):
            self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''					)					)

        with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''					):
            self.assert_dump_and_restore(
                tokenizer_r('''Small example to encode'''       , return_tensors=TensorType.NUMPY					)       , np.array_equal					)


    @require_tf
    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> str:
        import tensorflow as tf

        def tf_array_equals(lowercase__       , lowercase__					):
            return tf.reduce_all(tf.equal(lowercase__       , lowercase__					)					)

        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)
        __UpperCAmelCase			   =     BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

        with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''					):
            self.assert_dump_and_restore(
                tokenizer_p('''Small example to encode'''       , return_tensors=TensorType.TENSORFLOW					)       , lowercase__					)

        with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''					):
            self.assert_dump_and_restore(
                tokenizer_r('''Small example to encode'''       , return_tensors=TensorType.TENSORFLOW					)       , lowercase__					)


    @require_torch
    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> Tuple:
        import torch

        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)
        __UpperCAmelCase			   =     BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

        with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''					):
            self.assert_dump_and_restore(
                tokenizer_p('''Small example to encode'''       , return_tensors=TensorType.PYTORCH					)       , torch.equal					)

        with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''					):
            self.assert_dump_and_restore(
                tokenizer_r('''Small example to encode'''       , return_tensors=TensorType.PYTORCH					)       , torch.equal					)


    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> List[str]:
        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)
        __UpperCAmelCase			   =     BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

        with self.subTest('''Python Tokenizer'''					):
            self.assertFalse(tokenizer_p('''Small example to_encode'''					).is_fast					)

        with self.subTest('''Rust Tokenizer'''					):
            self.assertTrue(tokenizer_r('''Small example to_encode'''					).is_fast					)


    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> Tuple:
        __UpperCAmelCase			   =     BertTokenizerFast.from_pretrained('''bert-base-cased'''					)
        __UpperCAmelCase			   =     tokenizer_r(['''Test''', '''\xad''', '''test''']       , is_split_into_words=lowercase__					)

        self.assertEqual(encoded.word_to_tokens(0					)       , TokenSpan(start=1       , end=2					)					)
        self.assertEqual(encoded.word_to_tokens(1					)       , lowercase__					)
        self.assertEqual(encoded.word_to_tokens(2					)       , TokenSpan(start=2       , end=3					)					)


    def        lowerCAmelCase_  (self					)       -> Any:
        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''np'''					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (2, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (2,)					)
        # test converting the converted
        with CaptureStderr() as cs:
            __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''np'''					)
        self.assertFalse(len(cs.err					)       , msg=F'''should have no warning, but got {cs.err}'''					)

        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''np'''       , prepend_batch_axis=lowercase__					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (1, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (1,)					)


    @require_torch
    def        lowerCAmelCase_  (self					)       -> List[Any]:
        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''pt'''					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (2, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (2,)					)
        # test converting the converted
        with CaptureStderr() as cs:
            __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''pt'''					)
        self.assertFalse(len(cs.err					)       , msg=F'''should have no warning, but got {cs.err}'''					)

        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''pt'''       , prepend_batch_axis=lowercase__					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (1, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (1,)					)


    @require_tf
    def        lowerCAmelCase_  (self					)       -> Any:
        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''tf'''					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (2, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (2,)					)
        # test converting the converted
        with CaptureStderr() as cs:
            __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''tf'''					)
        self.assertFalse(len(cs.err					)       , msg=F'''should have no warning, but got {cs.err}'''					)

        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''tf'''       , prepend_batch_axis=lowercase__					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (1, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (1,)					)


    @require_flax
    def        lowerCAmelCase_  (self					)       -> Optional[Any]:
        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''jax'''					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (2, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (2,)					)
        # test converting the converted
        with CaptureStderr() as cs:
            __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''jax'''					)
        self.assertFalse(len(cs.err					)       , msg=F'''should have no warning, but got {cs.err}'''					)

        __UpperCAmelCase			   =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
        __UpperCAmelCase			   =     batch.convert_to_tensors(tensor_type='''jax'''       , prepend_batch_axis=lowercase__					)
        self.assertEqual(tensor_batch['''inputs'''].shape       , (1, 3)					)
        self.assertEqual(tensor_batch['''labels'''].shape       , (1,)					)


    def        lowerCAmelCase_  (self					)       -> str:
        __UpperCAmelCase			   =     [{'''input_ids''': np.array([0, 1, 2]					)}, {'''input_ids''': np.array([0, 1, 2, 3]					)}]
        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)

        __UpperCAmelCase			   =     tokenizer.pad(lowercase__       , padding=lowercase__					)
        self.assertTrue(isinstance(batch['''input_ids''']       , np.ndarray					)					)
        self.assertEqual(batch['''input_ids'''].tolist()       , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
        __UpperCAmelCase			   =     tokenizer.pad(lowercase__       , padding=lowercase__       , return_tensors='''np'''					)
        self.assertTrue(isinstance(batch['''input_ids''']       , np.ndarray					)					)
        self.assertEqual(batch['''input_ids'''].tolist()       , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)


    @require_torch
    def        lowerCAmelCase_  (self					)       -> List[Any]:
        import torch

        __UpperCAmelCase			   =     [{'''input_ids''': torch.tensor([0, 1, 2]					)}, {'''input_ids''': torch.tensor([0, 1, 2, 3]					)}]
        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)

        __UpperCAmelCase			   =     tokenizer.pad(lowercase__       , padding=lowercase__					)
        self.assertTrue(isinstance(batch['''input_ids''']       , torch.Tensor					)					)
        self.assertEqual(batch['''input_ids'''].tolist()       , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
        __UpperCAmelCase			   =     tokenizer.pad(lowercase__       , padding=lowercase__       , return_tensors='''pt'''					)
        self.assertTrue(isinstance(batch['''input_ids''']       , torch.Tensor					)					)
        self.assertEqual(batch['''input_ids'''].tolist()       , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)


    @require_tf
    def        lowerCAmelCase_  (self					)       -> Tuple:
        import tensorflow as tf

        __UpperCAmelCase			   =     [{'''input_ids''': tf.constant([0, 1, 2]					)}, {'''input_ids''': tf.constant([0, 1, 2, 3]					)}]
        __UpperCAmelCase			   =     BertTokenizer.from_pretrained('''bert-base-cased'''					)

        __UpperCAmelCase			   =     tokenizer.pad(lowercase__       , padding=lowercase__					)
        self.assertTrue(isinstance(batch['''input_ids''']       , tf.Tensor					)					)
        self.assertEqual(batch['''input_ids'''].numpy().tolist()       , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
        __UpperCAmelCase			   =     tokenizer.pad(lowercase__       , padding=lowercase__       , return_tensors='''tf'''					)
        self.assertTrue(isinstance(batch['''input_ids''']       , tf.Tensor					)					)
        self.assertEqual(batch['''input_ids'''].numpy().tolist()       , [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)


    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> List[str]:
        __UpperCAmelCase			   =     Tokenizer(WordPiece(unk_token='''[UNK]'''					)					)
        PreTrainedTokenizerFast(tokenizer_object=lowercase__					)






    @require_tokenizers
    def        lowerCAmelCase_  (self					)       -> List[Any]:
        __UpperCAmelCase			   =     Tokenizer(WordPiece(unk_token='''[UNK]'''					)					)
        with tempfile.TemporaryDirectory() as tmpdirname:
            bert_tokenizer.save(os.path.join(lowercase__       , '''tokenizer.json'''					)					)
            PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowercase__       , '''tokenizer.json'''					)					)






def  __a    (				SCREAMING_SNAKE_CASE ) ->       float:






    '''simple docstring'''
    return -num if num < 0 else num


def  __a    (				SCREAMING_SNAKE_CASE ) ->       int:






    '''simple docstring'''
    if len(SCREAMING_SNAKE_CASE ) == 0:
        raise ValueError('''abs_min() arg is an empty sequence''' )
    __UpperCAmelCase			   =     x[0]
    for i in x:
        if abs_val(SCREAMING_SNAKE_CASE ) < abs_val(SCREAMING_SNAKE_CASE ):
            __UpperCAmelCase			   =     i
    return j


def  __a    (				SCREAMING_SNAKE_CASE ) ->       int:






    '''simple docstring'''
    if len(SCREAMING_SNAKE_CASE ) == 0:
        raise ValueError('''abs_max() arg is an empty sequence''' )
    __UpperCAmelCase			   =     x[0]
    for i in x:
        if abs(SCREAMING_SNAKE_CASE ) > abs(SCREAMING_SNAKE_CASE ):
            __UpperCAmelCase			   =     i
    return j


def  __a    (				SCREAMING_SNAKE_CASE ) ->       int:






    '''simple docstring'''
    if len(SCREAMING_SNAKE_CASE ) == 0:
        raise ValueError('''abs_max_sort() arg is an empty sequence''' )
    return sorted(SCREAMING_SNAKE_CASE    ,				key=SCREAMING_SNAKE_CASE )[-1]


def  __a    (				) ->       Union[str, Any]:






    '''simple docstring'''
    assert abs_val(0 ) == 0
    assert abs_val(3_4 ) == 3_4
    assert abs_val(-1_0_0_0_0_0_0_0_0_0_0_0 ) == 1_0_0_0_0_0_0_0_0_0_0_0

    __UpperCAmelCase			   =     [-3, -1, 2, -1_1]
    assert abs_max(SCREAMING_SNAKE_CASE ) == -1_1
    assert abs_max_sort(SCREAMING_SNAKE_CASE ) == -1_1
    assert abs_min(SCREAMING_SNAKE_CASE ) == -1


if __name__ == "__main__":
       import doctest

       doctest.testmod()

       test_abs_val()
       print(abs_val(-34))  # --> 34




