



'''simple docstring'''




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece






class 						_UpperCamelCase  (							unittest.TestCase   ):


			'''simple docstring'''

			def UpperCamelCase__ (							self					:			List[Any]							,					lowerCAmelCase__					:			Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Dict   	=		list(tokenizer_class.max_model_input_sizes.keys()						)
					for model_name in sa_models[:1]:
							__SCREAMING_SNAKE_CASE	:		List[str]   	=		tokenizer_class.from_pretrained(lowerCAmelCase__						)
							self.assertIsNotNone(lowerCAmelCase__						)
							self.assertIsInstance(lowerCAmelCase__							,					lowerCAmelCase__						)
							self.assertIsInstance(lowerCAmelCase__							,					lowerCAmelCase__						)

							for special_tok in tokenizer.all_special_tokens:
									self.assertIsInstance(lowerCAmelCase__							,					lowerCAmelCase__						)
									__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		tokenizer.convert_tokens_to_ids(lowerCAmelCase__						)
									self.assertIsInstance(lowerCAmelCase__							,					lowerCAmelCase__						)

			def UpperCamelCase__ (							self					:			Union[str, Any]							,					lowerCAmelCase__					:			BatchEncoding							,					lowerCAmelCase__					:			Optional[Callable] = None						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		pickle.dumps(lowerCAmelCase__						)
					self.assertIsNotNone(lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE	:		int   	=		pickle.loads(lowerCAmelCase__						)

					# Ensure is_fast is correctly restored
					self.assertEqual(be_restored.is_fast							,					be_original.is_fast						)

					# Ensure encodings are potentially correctly restored
					if be_original.is_fast:
							self.assertIsNotNone(be_restored.encodings						)
					else:
							self.assertIsNone(be_restored.encodings						)

					# Ensure the keys are the same
					for original_v, restored_v in zip(be_original.values()							,					be_restored.values()						):
							if equal_op:
									self.assertTrue(equal_op(lowerCAmelCase__							,					lowerCAmelCase__						)						)
							else:
									self.assertEqual(lowerCAmelCase__							,					lowerCAmelCase__						)

			@slow
			def UpperCamelCase__ (							self					:			Union[str, Any]						):





					"""simple docstring"""






					self.check_tokenizer_from_pretrained(lowerCAmelCase__						)

			def UpperCamelCase__ (							self					:			Optional[int]						):





					"""simple docstring"""






					self.assertEqual(TensorType("""tf"""						)							,					TensorType.TENSORFLOW						)
					self.assertEqual(TensorType("""pt"""						)							,					TensorType.PYTORCH						)
					self.assertEqual(TensorType("""np"""						)							,					TensorType.NUMPY						)

			@require_tokenizers
			def UpperCamelCase__ (							self					:			Optional[int]						):





					"""simple docstring"""






					import numpy as np

					__SCREAMING_SNAKE_CASE	:		List[str]   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

					# Python no tensor
					with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
							self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

					with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
							self.assert_dump_and_restore(
							    tokenizer_p("""Small example to encode"""							,					return_tensors=TensorType.NUMPY						)							,					np.array_equal						)

					with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
							self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

					with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
							self.assert_dump_and_restore(
							    tokenizer_r("""Small example to encode"""							,					return_tensors=TensorType.NUMPY						)							,					np.array_equal						)

			@require_tf
			@require_tokenizers
			def UpperCamelCase__ (							self					:			str						):





					"""simple docstring"""






					import tensorflow as tf

					def tf_array_equals(lowerCAmelCase__					:			Dict							,					lowerCAmelCase__					:			Optional[int]						):
							return tf.reduce_all(tf.equal(lowerCAmelCase__							,					lowerCAmelCase__						)						)

					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

					with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
							self.assert_dump_and_restore(
							    tokenizer_p("""Small example to encode"""							,					return_tensors=TensorType.TENSORFLOW						)							,					lowerCAmelCase__						)

					with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
							self.assert_dump_and_restore(
							    tokenizer_r("""Small example to encode"""							,					return_tensors=TensorType.TENSORFLOW						)							,					lowerCAmelCase__						)

			@require_torch
			@require_tokenizers
			def UpperCamelCase__ (							self					:			Optional[Any]						):





					"""simple docstring"""






					import torch

					__SCREAMING_SNAKE_CASE	:		Tuple   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)
					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

					with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
							self.assert_dump_and_restore(
							    tokenizer_p("""Small example to encode"""							,					return_tensors=TensorType.PYTORCH						)							,					torch.equal						)

					with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
							self.assert_dump_and_restore(
							    tokenizer_r("""Small example to encode"""							,					return_tensors=TensorType.PYTORCH						)							,					torch.equal						)

			@require_tokenizers
			def UpperCamelCase__ (							self					:			str						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Dict   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

					with self.subTest("""Python Tokenizer"""						):
							self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

					with self.subTest("""Rust Tokenizer"""						):
							self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)

			@require_tokenizers
			def UpperCamelCase__ (							self					:			Tuple						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
					__SCREAMING_SNAKE_CASE	:		str   	=		tokenizer_r(["""Test""", """\xad""", """test"""]							,					is_split_into_words=lowerCAmelCase__						)

					self.assertEqual(encoded.word_to_tokens(0						)							,					TokenSpan(start=1							,					end=2						)						)
					self.assertEqual(encoded.word_to_tokens(1						)							,					lowerCAmelCase__						)
					self.assertEqual(encoded.word_to_tokens(2						)							,					TokenSpan(start=2							,					end=3						)						)

			def UpperCamelCase__ (							self					:			Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		batch.convert_to_tensors(tensor_type="""np"""						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(2, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(2,)						)
					# test converting the converted
					with CaptureStderr() as cs:
							__SCREAMING_SNAKE_CASE	:		List[Any]   	=		batch.convert_to_tensors(tensor_type="""np"""						)
					self.assertFalse(len(cs.err						)							,					msg=F"should have no warning, but got {cs.err}"						)

					__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		batch.convert_to_tensors(tensor_type="""np"""							,					prepend_batch_axis=lowerCAmelCase__						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(1, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(1,)						)

			@require_torch
			def UpperCamelCase__ (							self					:			Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Tuple   	=		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
					__SCREAMING_SNAKE_CASE	:		str   	=		batch.convert_to_tensors(tensor_type="""pt"""						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(2, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(2,)						)
					# test converting the converted
					with CaptureStderr() as cs:
							__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		batch.convert_to_tensors(tensor_type="""pt"""						)
					self.assertFalse(len(cs.err						)							,					msg=F"should have no warning, but got {cs.err}"						)

					__SCREAMING_SNAKE_CASE	:		str   	=		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		batch.convert_to_tensors(tensor_type="""pt"""							,					prepend_batch_axis=lowerCAmelCase__						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(1, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(1,)						)

			@require_tf
			def UpperCamelCase__ (							self					:			Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
					__SCREAMING_SNAKE_CASE	:		str   	=		batch.convert_to_tensors(tensor_type="""tf"""						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(2, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(2,)						)
					# test converting the converted
					with CaptureStderr() as cs:
							__SCREAMING_SNAKE_CASE	:		Any   	=		batch.convert_to_tensors(tensor_type="""tf"""						)
					self.assertFalse(len(cs.err						)							,					msg=F"should have no warning, but got {cs.err}"						)

					__SCREAMING_SNAKE_CASE	:		Any   	=		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
					__SCREAMING_SNAKE_CASE	:		str   	=		batch.convert_to_tensors(tensor_type="""tf"""							,					prepend_batch_axis=lowerCAmelCase__						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(1, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(1,)						)

			@require_flax
			def UpperCamelCase__ (							self					:			Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		Dict   	=		BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
					__SCREAMING_SNAKE_CASE	:		int   	=		batch.convert_to_tensors(tensor_type="""jax"""						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(2, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(2,)						)
					# test converting the converted
					with CaptureStderr() as cs:
							__SCREAMING_SNAKE_CASE	:		Any   	=		batch.convert_to_tensors(tensor_type="""jax"""						)
					self.assertFalse(len(cs.err						)							,					msg=F"should have no warning, but got {cs.err}"						)

					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
					__SCREAMING_SNAKE_CASE	:		Dict   	=		batch.convert_to_tensors(tensor_type="""jax"""							,					prepend_batch_axis=lowerCAmelCase__						)
					self.assertEqual(tensor_batch["""inputs"""].shape							,					(1, 3)						)
					self.assertEqual(tensor_batch["""labels"""].shape							,					(1,)						)

			def UpperCamelCase__ (							self					:			List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		int   	=		[{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)

					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		tokenizer.pad(lowerCAmelCase__							,					padding=lowerCAmelCase__						)
					self.assertTrue(isinstance(batch["""input_ids"""]							,					np.ndarray						)						)
					self.assertEqual(batch["""input_ids"""].tolist()							,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
					__SCREAMING_SNAKE_CASE	:		int   	=		tokenizer.pad(lowerCAmelCase__							,					padding=lowerCAmelCase__							,					return_tensors="""np"""						)
					self.assertTrue(isinstance(batch["""input_ids"""]							,					np.ndarray						)						)
					self.assertEqual(batch["""input_ids"""].tolist()							,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)

			@require_torch
			def UpperCamelCase__ (							self					:			Optional[Any]						):





					"""simple docstring"""






					import torch

					__SCREAMING_SNAKE_CASE	:		str   	=		[{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
					__SCREAMING_SNAKE_CASE	:		Dict   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)

					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		tokenizer.pad(lowerCAmelCase__							,					padding=lowerCAmelCase__						)
					self.assertTrue(isinstance(batch["""input_ids"""]							,					torch.Tensor						)						)
					self.assertEqual(batch["""input_ids"""].tolist()							,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
					__SCREAMING_SNAKE_CASE	:		int   	=		tokenizer.pad(lowerCAmelCase__							,					padding=lowerCAmelCase__							,					return_tensors="""pt"""						)
					self.assertTrue(isinstance(batch["""input_ids"""]							,					torch.Tensor						)						)
					self.assertEqual(batch["""input_ids"""].tolist()							,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)

			@require_tf
			def UpperCamelCase__ (							self					:			List[Any]						):





					"""simple docstring"""






					import tensorflow as tf

					__SCREAMING_SNAKE_CASE	:		Tuple   	=		[{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		BertTokenizer.from_pretrained("""bert-base-cased"""						)

					__SCREAMING_SNAKE_CASE	:		int   	=		tokenizer.pad(lowerCAmelCase__							,					padding=lowerCAmelCase__						)
					self.assertTrue(isinstance(batch["""input_ids"""]							,					tf.Tensor						)						)
					self.assertEqual(batch["""input_ids"""].numpy().tolist()							,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		tokenizer.pad(lowerCAmelCase__							,					padding=lowerCAmelCase__							,					return_tensors="""tf"""						)
					self.assertTrue(isinstance(batch["""input_ids"""]							,					tf.Tensor						)						)
					self.assertEqual(batch["""input_ids"""].numpy().tolist()							,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)

			@require_tokenizers
			def UpperCamelCase__ (							self					:			Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		List[str]   	=		Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
					PreTrainedTokenizerFast(tokenizer_object=lowerCAmelCase__						)



			@require_tokenizers
			def UpperCamelCase__ (							self					:			int						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE	:		str   	=		Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
					with tempfile.TemporaryDirectory() as tmpdirname:
							bert_tokenizer.save(os.path.join(lowerCAmelCase__							,					"""tokenizer.json"""						)						)
							PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCAmelCase__							,					"""tokenizer.json"""						)						)




'''simple docstring'''




UpperCamelCase__	:     List[Any]    =			10_00


def     lowerCAmelCase_       (     _lowerCamelCase:       float ):
		return x * x


def     lowerCAmelCase_       (     _lowerCamelCase:       Any	,				_lowerCamelCase:       float	,				_lowerCamelCase:       float	,				_lowerCamelCase:       int = 4 ):
		assert callable(
		    _lowerCamelCase ), F"the function(object) passed should be callable your input : {function}"
		assert isinstance(_lowerCamelCase	,				(float, int) ), F"a should be float or integer your input : {a}"
		assert isinstance(function(_lowerCamelCase )	,				(float, int) ), (
		    "the function should return integer or float return type of your function, "
		    F"{type(_lowerCamelCase )}"
		)
		assert isinstance(_lowerCamelCase	,				(float, int) ), F"b should be float or integer your input : {b}"
		assert (
		    isinstance(_lowerCamelCase	,				_lowerCamelCase ) and precision > 0
		), F"precision should be positive integer your input : {precision}"

		# just applying the formula of simpson for approximate integration written in
		# mentioned article in first comment of this file and above this function

		__SCREAMING_SNAKE_CASE	:		List[Any]   	=		(b - a) / N_STEPS
		__SCREAMING_SNAKE_CASE	:		List[Any]   	=		function(_lowerCamelCase ) + function(_lowerCamelCase )

		for i in range(1	,				_lowerCamelCase ):
				__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		a + h * i
				result += function(_lowerCamelCase ) * (4 if i % 2 else 2)

		result *= h / 3
		return round(_lowerCamelCase	,				_lowerCamelCase )


if __name__ == "__main__":
						import doctest

						doctest.testmod()