
'''simple docstring'''


import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
					from tokenizers import Tokenizer
					from tokenizers.models import WordPiece

class 		__lowercase			(					unittest.TestCase      ):

			def     UpperCAmelCase__	(self     ,				A ):
								lowerCamelCase_   : Dict       =						list(tokenizer_class.max_model_input_sizes.keys() )
								for model_name in sa_models[:1]:
													lowerCamelCase_   : Optional[int]       =						tokenizer_class.from_pretrained(A )
													self.assertIsNotNone(A )
													self.assertIsInstance(A     ,				A )
													self.assertIsInstance(A     ,				A )

													for special_tok in tokenizer.all_special_tokens:
																		self.assertIsInstance(A     ,				A )
																		lowerCamelCase_   : List[Any]       =						tokenizer.convert_tokens_to_ids(A )
																		self.assertIsInstance(A     ,				A )

			def     UpperCAmelCase__	(self     ,				A     ,				A = None ):
								lowerCamelCase_   : Tuple       =						pickle.dumps(A )
								self.assertIsNotNone(A )

								lowerCamelCase_   : Optional[int]       =						pickle.loads(A )

								# Ensure is_fast is correctly restored
								self.assertEqual(be_restored.is_fast     ,				be_original.is_fast )

								# Ensure encodings are potentially correctly restored
								if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings )
								else:
													self.assertIsNone(be_restored.encodings )

								# Ensure the keys are the same
								for original_v, restored_v in zip(be_original.values()     ,				be_restored.values() ):
													if equal_op:
																		self.assertTrue(equal_op(A     ,				A ) )
													else:
																		self.assertEqual(A     ,				A )

			@slow
			def     UpperCAmelCase__	(self ):
								self.check_tokenizer_from_pretrained(A )

			def     UpperCAmelCase__	(self ):
								self.assertEqual(TensorType('''tf''' )     ,				TensorType.TENSORFLOW )
								self.assertEqual(TensorType('''pt''' )     ,				TensorType.PYTORCH )
								self.assertEqual(TensorType('''np''' )     ,				TensorType.NUMPY )

			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								import numpy as np

								lowerCamelCase_   : Dict       =						BertTokenizer.from_pretrained('''bert-base-cased''' )
								lowerCamelCase_   : Optional[Any]       =						BertTokenizerFast.from_pretrained('''bert-base-cased''' )

								# Python no tensor
								with self.subTest('''BatchEncoding (Python, return_tensors=None)''' ):
													self.assert_dump_and_restore(tokenizer_p('''Small example to encode''' ) )

								with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)''' ):
													self.assert_dump_and_restore(
													    tokenizer_p('''Small example to encode'''     ,				return_tensors=TensorType.NUMPY )     ,				np.array_equal )

								with self.subTest('''BatchEncoding (Rust, return_tensors=None)''' ):
													self.assert_dump_and_restore(tokenizer_r('''Small example to encode''' ) )

								with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)''' ):
													self.assert_dump_and_restore(
													    tokenizer_r('''Small example to encode'''     ,				return_tensors=TensorType.NUMPY )     ,				np.array_equal )

			@require_tf
			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								import tensorflow as tf

								def tf_array_equals(A     ,				A ):
													return tf.reduce_all(tf.equal(A     ,				A ) )

								lowerCamelCase_   : Tuple       =						BertTokenizer.from_pretrained('''bert-base-cased''' )
								lowerCamelCase_   : Dict       =						BertTokenizerFast.from_pretrained('''bert-base-cased''' )

								with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)''' ):
													self.assert_dump_and_restore(
													    tokenizer_p('''Small example to encode'''     ,				return_tensors=TensorType.TENSORFLOW )     ,				A )

								with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)''' ):
													self.assert_dump_and_restore(
													    tokenizer_r('''Small example to encode'''     ,				return_tensors=TensorType.TENSORFLOW )     ,				A )

			@require_torch
			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								import torch

								lowerCamelCase_   : Dict       =						BertTokenizer.from_pretrained('''bert-base-cased''' )
								lowerCamelCase_   : List[str]       =						BertTokenizerFast.from_pretrained('''bert-base-cased''' )

								with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)''' ):
													self.assert_dump_and_restore(
													    tokenizer_p('''Small example to encode'''     ,				return_tensors=TensorType.PYTORCH )     ,				torch.equal )

								with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)''' ):
													self.assert_dump_and_restore(
													    tokenizer_r('''Small example to encode'''     ,				return_tensors=TensorType.PYTORCH )     ,				torch.equal )

			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Any       =						BertTokenizer.from_pretrained('''bert-base-cased''' )
								lowerCamelCase_   : Any       =						BertTokenizerFast.from_pretrained('''bert-base-cased''' )

								with self.subTest('''Python Tokenizer''' ):
													self.assertFalse(tokenizer_p('''Small example to_encode''' ).is_fast )

								with self.subTest('''Rust Tokenizer''' ):
													self.assertTrue(tokenizer_r('''Small example to_encode''' ).is_fast )

			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : List[Any]       =						BertTokenizerFast.from_pretrained('''bert-base-cased''' )
								lowerCamelCase_   : Optional[int]       =						tokenizer_r(['''Test''', '''\xad''', '''test''']     ,				is_split_into_words=A )

								self.assertEqual(encoded.word_to_tokens(0 )     ,				TokenSpan(start=1     ,				end=2 ) )
								self.assertEqual(encoded.word_to_tokens(1 )     ,				A )
								self.assertEqual(encoded.word_to_tokens(2 )     ,				TokenSpan(start=2     ,				end=3 ) )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Optional[Any]       =						BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
								lowerCamelCase_   : Optional[int]       =						batch.convert_to_tensors(tensor_type='''np''' )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(2, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(2,) )
								# test converting the converted
								with CaptureStderr() as cs:
													lowerCamelCase_   : List[Any]       =						batch.convert_to_tensors(tensor_type='''np''' )
								self.assertFalse(len(cs.err )     ,				msg=F"""should have no warning, but got {cs.err}""" )

								lowerCamelCase_   : Tuple       =						BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
								lowerCamelCase_   : Tuple       =						batch.convert_to_tensors(tensor_type='''np'''     ,				prepend_batch_axis=A )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(1, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(1,) )

			@require_torch
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Optional[Any]       =						BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
								lowerCamelCase_   : str       =						batch.convert_to_tensors(tensor_type='''pt''' )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(2, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(2,) )
								# test converting the converted
								with CaptureStderr() as cs:
													lowerCamelCase_   : str       =						batch.convert_to_tensors(tensor_type='''pt''' )
								self.assertFalse(len(cs.err )     ,				msg=F"""should have no warning, but got {cs.err}""" )

								lowerCamelCase_   : str       =						BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
								lowerCamelCase_   : Union[str, Any]       =						batch.convert_to_tensors(tensor_type='''pt'''     ,				prepend_batch_axis=A )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(1, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(1,) )

			@require_tf
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Optional[int]       =						BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
								lowerCamelCase_   : int       =						batch.convert_to_tensors(tensor_type='''tf''' )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(2, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(2,) )
								# test converting the converted
								with CaptureStderr() as cs:
													lowerCamelCase_   : Dict       =						batch.convert_to_tensors(tensor_type='''tf''' )
								self.assertFalse(len(cs.err )     ,				msg=F"""should have no warning, but got {cs.err}""" )

								lowerCamelCase_   : str       =						BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
								lowerCamelCase_   : str       =						batch.convert_to_tensors(tensor_type='''tf'''     ,				prepend_batch_axis=A )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(1, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(1,) )

			@require_flax
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Tuple       =						BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]} )
								lowerCamelCase_   : int       =						batch.convert_to_tensors(tensor_type='''jax''' )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(2, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(2,) )
								# test converting the converted
								with CaptureStderr() as cs:
													lowerCamelCase_   : List[str]       =						batch.convert_to_tensors(tensor_type='''jax''' )
								self.assertFalse(len(cs.err )     ,				msg=F"""should have no warning, but got {cs.err}""" )

								lowerCamelCase_   : Union[str, Any]       =						BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0} )
								lowerCamelCase_   : Any       =						batch.convert_to_tensors(tensor_type='''jax'''     ,				prepend_batch_axis=A )
								self.assertEqual(tensor_batch['''inputs'''].shape     ,				(1, 3) )
								self.assertEqual(tensor_batch['''labels'''].shape     ,				(1,) )

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Dict       =						[{'''input_ids''': np.array([0, 1, 2] )}, {'''input_ids''': np.array([0, 1, 2, 3] )}]
								lowerCamelCase_   : Tuple       =						BertTokenizer.from_pretrained('''bert-base-cased''' )

								lowerCamelCase_   : Union[str, Any]       =						tokenizer.pad(A     ,				padding=A )
								self.assertTrue(isinstance(batch['''input_ids''']     ,				np.ndarray ) )
								self.assertEqual(batch['''input_ids'''].tolist()     ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
								lowerCamelCase_   : Dict       =						tokenizer.pad(A     ,				padding=A     ,				return_tensors='''np''' )
								self.assertTrue(isinstance(batch['''input_ids''']     ,				np.ndarray ) )
								self.assertEqual(batch['''input_ids'''].tolist()     ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )

			@require_torch
			def     UpperCAmelCase__	(self ):
								import torch

								lowerCamelCase_   : int       =						[{'''input_ids''': torch.tensor([0, 1, 2] )}, {'''input_ids''': torch.tensor([0, 1, 2, 3] )}]
								lowerCamelCase_   : Dict       =						BertTokenizer.from_pretrained('''bert-base-cased''' )

								lowerCamelCase_   : Tuple       =						tokenizer.pad(A     ,				padding=A )
								self.assertTrue(isinstance(batch['''input_ids''']     ,				torch.Tensor ) )
								self.assertEqual(batch['''input_ids'''].tolist()     ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
								lowerCamelCase_   : Dict       =						tokenizer.pad(A     ,				padding=A     ,				return_tensors='''pt''' )
								self.assertTrue(isinstance(batch['''input_ids''']     ,				torch.Tensor ) )
								self.assertEqual(batch['''input_ids'''].tolist()     ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )

			@require_tf
			def     UpperCAmelCase__	(self ):
								import tensorflow as tf

								lowerCamelCase_   : Optional[int]       =						[{'''input_ids''': tf.constant([0, 1, 2] )}, {'''input_ids''': tf.constant([0, 1, 2, 3] )}]
								lowerCamelCase_   : Dict       =						BertTokenizer.from_pretrained('''bert-base-cased''' )

								lowerCamelCase_   : Optional[Any]       =						tokenizer.pad(A     ,				padding=A )
								self.assertTrue(isinstance(batch['''input_ids''']     ,				tf.Tensor ) )
								self.assertEqual(batch['''input_ids'''].numpy().tolist()     ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
								lowerCamelCase_   : Optional[int]       =						tokenizer.pad(A     ,				padding=A     ,				return_tensors='''tf''' )
								self.assertTrue(isinstance(batch['''input_ids''']     ,				tf.Tensor ) )
								self.assertEqual(batch['''input_ids'''].numpy().tolist()     ,				[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )

			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : List[Any]       =						Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
								PreTrainedTokenizerFast(tokenizer_object=A )






			@require_tokenizers
			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : List[Any]       =						Tokenizer(WordPiece(unk_token='''[UNK]''' ) )
								with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(A     ,				'''tokenizer.json''' ) )
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(A     ,				'''tokenizer.json''' ) )




'''simple docstring'''


import argparse
import logging
import sys
from unittest.mock import patch

import run_glue_with_pabee

from transformers.testing_utils import TestCasePlus


logging.basicConfig(level=logging.DEBUG)

__lowercase							:				Optional[Any]					=       logging.getLogger()




def lowercase_		(				)       ->							Union[str, Any]:
					'''simple docstring'''

					lowerCamelCase_   : Any       =						argparse.ArgumentParser()
					parser.add_argument('''-f'''			)
					lowerCamelCase_   : Optional[int]       =						parser.parse_args()
					return args.f






class 		__lowercase			(					_lowercase      ):

			def     UpperCAmelCase__	(self ):
								lowerCamelCase_   : Dict       =						logging.StreamHandler(sys.stdout )
								logger.addHandler(A )

								lowerCamelCase_   : int       =						self.get_auto_remove_tmp_dir()
								lowerCamelCase_   : Dict       =						F"""
            run_glue_with_pabee.py
            --model_type albert
            --model_name_or_path albert-base-v2
            --data_dir ./tests/fixtures/tests_samples/MRPC/
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --task_name mrpc
            --do_train
            --do_eval
            --per_gpu_train_batch_size=2
            --per_gpu_eval_batch_size=1
            --learning_rate=2e-5
            --max_steps=50
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            """.split()

								with patch.object(A     ,				'''argv'''     ,				A ):
													lowerCamelCase_   : Optional[Any]       =						run_glue_with_pabee.main()
													for value in result.values():
																		self.assertGreaterEqual(A     ,				0.75 )


