

"""simple docstring"""




import tempfile
import unittest

from transformers import DPRConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import DPRContextEncoder, DPRQuestionEncoder, DPRReader, DPRReaderTokenizer
						from transformers.models.dpr.modeling_dpr import (
						    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
						    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
						    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
						)





class 				lowerCamelCase  :

			def __init__(   self			:  str      ,			__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  str=1_3      ,			__UpperCAmelCase			:  str=7      ,			__UpperCAmelCase			:  Tuple=False      ,			__UpperCAmelCase			:  str=True      ,			__UpperCAmelCase			:  Dict=True      ,			__UpperCAmelCase			:  List[str]=True      ,			__UpperCAmelCase			:  Union[str, Any]=9_9      ,			__UpperCAmelCase			:  List[str]=3_2      ,			__UpperCAmelCase			:  Tuple=5      ,			__UpperCAmelCase			:  Tuple=4      ,			__UpperCAmelCase			:  List[Any]=3_7      ,			__UpperCAmelCase			:  int="gelu"      ,			__UpperCAmelCase			:  List[str]=0.1      ,			__UpperCAmelCase			:  Union[str, Any]=0.1      ,			__UpperCAmelCase			:  Any=5_1_2      ,			__UpperCAmelCase			:  List[Any]=1_6      ,			__UpperCAmelCase			:  List[Any]=2      ,			__UpperCAmelCase			:  int=0.02      ,			__UpperCAmelCase			:  Union[str, Any]=3      ,			__UpperCAmelCase			:  Optional[int]=4      ,			__UpperCAmelCase			:  str=None      ,			__UpperCAmelCase			:  Union[str, Any]=0      ,			)				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= parent
						SCREAMING_SNAKE_CASE__									= batch_size
						SCREAMING_SNAKE_CASE__									= seq_length
						SCREAMING_SNAKE_CASE__									= is_training
						SCREAMING_SNAKE_CASE__									= use_input_mask
						SCREAMING_SNAKE_CASE__									= use_token_type_ids
						SCREAMING_SNAKE_CASE__									= use_labels
						SCREAMING_SNAKE_CASE__									= vocab_size
						SCREAMING_SNAKE_CASE__									= hidden_size
						SCREAMING_SNAKE_CASE__									= num_hidden_layers
						SCREAMING_SNAKE_CASE__									= num_attention_heads
						SCREAMING_SNAKE_CASE__									= intermediate_size
						SCREAMING_SNAKE_CASE__									= hidden_act
						SCREAMING_SNAKE_CASE__									= hidden_dropout_prob
						SCREAMING_SNAKE_CASE__									= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE__									= max_position_embeddings
						SCREAMING_SNAKE_CASE__									= type_vocab_size
						SCREAMING_SNAKE_CASE__									= type_sequence_label_size
						SCREAMING_SNAKE_CASE__									= initializer_range
						SCREAMING_SNAKE_CASE__									= num_labels
						SCREAMING_SNAKE_CASE__									= num_choices
						SCREAMING_SNAKE_CASE__									= scope
						SCREAMING_SNAKE_CASE__									= projection_dim

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      str:
						SCREAMING_SNAKE_CASE__									= ids_tensor([self.batch_size, self.seq_length]      ,			self.vocab_size    )

						SCREAMING_SNAKE_CASE__									= None
						if self.use_input_mask:
									SCREAMING_SNAKE_CASE__									= random_attention_mask([self.batch_size, self.seq_length]    )

						SCREAMING_SNAKE_CASE__									= None
						if self.use_token_type_ids:
									SCREAMING_SNAKE_CASE__									= ids_tensor([self.batch_size, self.seq_length]      ,			self.type_vocab_size    )

						SCREAMING_SNAKE_CASE__									= None
						SCREAMING_SNAKE_CASE__									= None
						SCREAMING_SNAKE_CASE__									= None
						if self.use_labels:
									SCREAMING_SNAKE_CASE__									= ids_tensor([self.batch_size]      ,			self.type_sequence_label_size    )
									SCREAMING_SNAKE_CASE__									= ids_tensor([self.batch_size, self.seq_length]      ,			self.num_labels    )
									SCREAMING_SNAKE_CASE__									= ids_tensor([self.batch_size]      ,			self.num_choices    )

						SCREAMING_SNAKE_CASE__									= self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Any:
						return DPRConfig(
						    projection_dim=self.projection_dim      ,			vocab_size=self.vocab_size      ,			hidden_size=self.hidden_size      ,			num_hidden_layers=self.num_hidden_layers      ,			num_attention_heads=self.num_attention_heads      ,			intermediate_size=self.intermediate_size      ,			hidden_act=self.hidden_act      ,			hidden_dropout_prob=self.hidden_dropout_prob      ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,			max_position_embeddings=self.max_position_embeddings      ,			type_vocab_size=self.type_vocab_size      ,			initializer_range=self.initializer_range      ,			)

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]      ,			__UpperCAmelCase			:  str      ,			__UpperCAmelCase			:  List[Any]      ,			__UpperCAmelCase			:  List[Any]      ,			__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  Any      ,			__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  List[str]    )				->      Any:
						SCREAMING_SNAKE_CASE__									= DPRContextEncoder(config=lowerCAmelCase__    )
						model.to(lowerCAmelCase__    )
						model.eval()
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__      ,			attention_mask=lowerCAmelCase__      ,			token_type_ids=lowerCAmelCase__    )
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__      ,			token_type_ids=lowerCAmelCase__    )
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__    )
						self.parent.assertEqual(result.pooler_output.shape      ,			(self.batch_size, self.projection_dim or self.hidden_size)    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]      ,			__UpperCAmelCase			:  Any      ,			__UpperCAmelCase			:  int      ,			__UpperCAmelCase			:  List[Any]      ,			__UpperCAmelCase			:  str      ,			__UpperCAmelCase			:  Tuple      ,			__UpperCAmelCase			:  Tuple      ,			__UpperCAmelCase			:  Dict    )				->      Dict:
						SCREAMING_SNAKE_CASE__									= DPRQuestionEncoder(config=lowerCAmelCase__    )
						model.to(lowerCAmelCase__    )
						model.eval()
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__      ,			attention_mask=lowerCAmelCase__      ,			token_type_ids=lowerCAmelCase__    )
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__      ,			token_type_ids=lowerCAmelCase__    )
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__    )
						self.parent.assertEqual(result.pooler_output.shape      ,			(self.batch_size, self.projection_dim or self.hidden_size)    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[str]      ,			__UpperCAmelCase			:  List[str]      ,			__UpperCAmelCase			:  Union[str, Any]      ,			__UpperCAmelCase			:  Tuple      ,			__UpperCAmelCase			:  Tuple      ,			__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  Any      ,			__UpperCAmelCase			:  Union[str, Any]    )				->      Union[str, Any]:
						SCREAMING_SNAKE_CASE__									= DPRReader(config=lowerCAmelCase__    )
						model.to(lowerCAmelCase__    )
						model.eval()
						SCREAMING_SNAKE_CASE__									= model(
						    lowerCAmelCase__      ,			attention_mask=lowerCAmelCase__      ,			)

						self.parent.assertEqual(result.start_logits.shape      ,			(self.batch_size, self.seq_length)    )
						self.parent.assertEqual(result.end_logits.shape      ,			(self.batch_size, self.seq_length)    )
						self.parent.assertEqual(result.relevance_logits.shape      ,			(self.batch_size,)    )





			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE__ 
						)							,						(
						    SCREAMING_SNAKE_CASE__ 
						)							,						(
						    SCREAMING_SNAKE_CASE__ 
						)							,						(
						    SCREAMING_SNAKE_CASE__ 
						)							,						(
						    SCREAMING_SNAKE_CASE__ 
						)							,						(
						    SCREAMING_SNAKE_CASE__ 
						)							,						(
						    SCREAMING_SNAKE_CASE__ 
						)							,						 
						)									= config_and_inputs
						SCREAMING_SNAKE_CASE__									= {"""input_ids""": input_ids}
						return config, inputs_dict





@require_torch
class 				lowerCamelCase  (lowercase__     ,lowercase__     ,unittest.TestCase     ):
			lowerCamelCase__				:						List[Any]   						=							(
			    (
			        DPRContextEncoder,
			        DPRQuestionEncoder,
			        DPRReader,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCamelCase__				:						Optional[int]   						=							{'''feature-extraction''': DPRQuestionEncoder} if is_torch_available() else {}

			lowerCamelCase__				:						str   						=							False
			lowerCamelCase__				:						Tuple   						=							False  # why?
			lowerCamelCase__				:						Optional[Any]   						=							False
			lowerCamelCase__				:						Optional[int]   						=							False

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= DPRModelTester(self    )
						SCREAMING_SNAKE_CASE__									= ConfigTester(self      ,			config_class=lowerCAmelCase__      ,			hidden_size=3_7    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      List[Any]:
						self.config_tester.run_common_tests()

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      str:
						SCREAMING_SNAKE_CASE__									= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_context_encoder(*lowerCAmelCase__    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_question_encoder(*lowerCAmelCase__    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_reader(*lowerCAmelCase__    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= self.model_tester.prepare_config_and_inputs()[0]

						SCREAMING_SNAKE_CASE__									= DPRQuestionEncoder(config=lowerCAmelCase__    )
						model.to(lowerCAmelCase__    )
						model.eval()

						with tempfile.TemporaryDirectory() as tmp_dirname:
									model.save_pretrained(lowerCAmelCase__    )
									SCREAMING_SNAKE_CASE__									= DPRQuestionEncoder.from_pretrained(lowerCAmelCase__      ,			projection_dim=5_1_2    )

						self.assertIsNotNone(lowerCAmelCase__    )





			@slow
			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      List[str]:
						for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE__									= DPRContextEncoder.from_pretrained(lowerCAmelCase__    )
									self.assertIsNotNone(lowerCAmelCase__    )

						for model_name in DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE__									= DPRContextEncoder.from_pretrained(lowerCAmelCase__    )
									self.assertIsNotNone(lowerCAmelCase__    )

						for model_name in DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE__									= DPRQuestionEncoder.from_pretrained(lowerCAmelCase__    )
									self.assertIsNotNone(lowerCAmelCase__    )

						for model_name in DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE__									= DPRReader.from_pretrained(lowerCAmelCase__    )
									self.assertIsNotNone(lowerCAmelCase__    )





@require_torch
class 				lowerCamelCase  (unittest.TestCase     ):

			@slow
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= DPRQuestionEncoder.from_pretrained("""facebook/dpr-question_encoder-single-nq-base"""      ,			return_dict=lowerCAmelCase__    )
						model.to(lowerCAmelCase__    )

						SCREAMING_SNAKE_CASE__									= torch.tensor(
						    [[1_0_1, 7_5_9_2, 1_0_1_0, 2_0_0_3, 2_0_2_6, 3_8_9_9, 1_0_1_4_0, 1_0_2_9, 1_0_2]]      ,			dtype=torch.long      ,			device=lowerCAmelCase__    )  # [CLS] hello, is my dog cute? [SEP]
						SCREAMING_SNAKE_CASE__									= model(lowerCAmelCase__    )[0]  # embedding shape = (1, 768)
						# compare the actual values for a slice.
						SCREAMING_SNAKE_CASE__									= torch.tensor(
						    [
						        [
						            0.03_236_253,
						            0.12_753_335,
						            0.16_818_509,
						            0.00_279_786,
						            0.3_896_933,
						            0.24_264_945,
						            0.2_178_971,
						            -0.02_335_227,
						            -0.08_481_959,
						            -0.14_324_117,
						        ]
						    ]      ,			dtype=torch.float      ,			device=lowerCAmelCase__      ,			)
						self.assertTrue(torch.allclose(output[:, :1_0]      ,			lowerCAmelCase__      ,			atol=1e-4    )    )





			@slow
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= DPRReaderTokenizer.from_pretrained("""facebook/dpr-reader-single-nq-base"""    )
						SCREAMING_SNAKE_CASE__									= DPRReader.from_pretrained("""facebook/dpr-reader-single-nq-base"""    )
						model.to(lowerCAmelCase__    )

						SCREAMING_SNAKE_CASE__									= tokenizer(
						    questions="""What is love ?"""      ,			titles="""Haddaway"""      ,			texts="""What Is Love is a song recorded by the artist Haddaway"""      ,			padding=lowerCAmelCase__      ,			return_tensors="""pt"""      ,			)
						encoded_inputs.to(lowerCAmelCase__    )

						SCREAMING_SNAKE_CASE__									= model(**lowerCAmelCase__    )

						# compare the actual values for a slice.
						SCREAMING_SNAKE_CASE__									= torch.tensor(
						    [[-10.3_005, -10.7_765, -11.4_872, -11.6_841, -11.9_312, -10.3_002, -9.8_544, -11.7_378, -12.0_821, -10.2_975]]      ,			dtype=torch.float      ,			device=lowerCAmelCase__      ,			)

						SCREAMING_SNAKE_CASE__									= torch.tensor(
						    [[-11.0_684, -11.7_041, -11.5_397, -10.3_465, -10.8_791, -6.8_443, -11.9_959, -11.0_364, -10.0_096, -6.8_405]]      ,			dtype=torch.float      ,			device=lowerCAmelCase__      ,			)
						self.assertTrue(torch.allclose(outputs.start_logits[:, :1_0]      ,			lowerCAmelCase__      ,			atol=1e-4    )    )
						self.assertTrue(torch.allclose(outputs.end_logits[:, :1_0]      ,			lowerCAmelCase__      ,			atol=1e-4    )    )





"""simple docstring"""




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 				lowerCamelCase  (unittest.TestCase     ):

			def    SCREAMING_SNAKE_CASE			(   self			:  Union[str, Any]      ,			__UpperCAmelCase			:  str    )				->      int:
						SCREAMING_SNAKE_CASE__									= list(tokenizer_class.max_model_input_sizes.keys()    )
						for model_name in sa_models[:1]:
									SCREAMING_SNAKE_CASE__									= tokenizer_class.from_pretrained(__UpperCAmelCase    )
									self.assertIsNotNone(__UpperCAmelCase    )
									self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )
									self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )

									for special_tok in tokenizer.all_special_tokens:
												self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )
												SCREAMING_SNAKE_CASE__									= tokenizer.convert_tokens_to_ids(__UpperCAmelCase    )
												self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Dict      ,			__UpperCAmelCase			:  BatchEncoding      ,			__UpperCAmelCase			:  Optional[Callable] = None    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= pickle.dumps(__UpperCAmelCase    )
						self.assertIsNotNone(__UpperCAmelCase    )

						SCREAMING_SNAKE_CASE__									= pickle.loads(__UpperCAmelCase    )

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast      ,			be_original.is_fast    )

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
									self.assertIsNotNone(be_restored.encodings    )
						else:
									self.assertIsNone(be_restored.encodings    )

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()      ,			be_restored.values()    ):
									if equal_op:
												self.assertTrue(equal_op(__UpperCAmelCase      ,			__UpperCAmelCase    )    )
									else:
												self.assertEqual(__UpperCAmelCase      ,			__UpperCAmelCase    )

			@slow
			def    SCREAMING_SNAKE_CASE			(   self			:  str    )				->      Optional[Any]:
						self.check_tokenizer_from_pretrained(__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      Optional[int]:
						self.assertEqual(TensorType("""tf"""    )      ,			TensorType.TENSORFLOW    )
						self.assertEqual(TensorType("""pt"""    )      ,			TensorType.PYTORCH    )
						self.assertEqual(TensorType("""np"""    )      ,			TensorType.NUMPY    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      List[str]:
						import numpy as np

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						# Python no tensor
						with self.subTest("""BatchEncoding (Python, return_tensors=None)"""    ):
									self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""    )    )

						with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.NUMPY    )      ,			np.array_equal    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""    ):
									self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""    )    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.NUMPY    )      ,			np.array_equal    )

			@require_tf
			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      int:
						import tensorflow as tf

						def tf_array_equals(__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  str    ):
									return tf.reduce_all(tf.equal(__UpperCAmelCase      ,			__UpperCAmelCase    )    )

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.TENSORFLOW    )      ,			__UpperCAmelCase    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.TENSORFLOW    )      ,			__UpperCAmelCase    )

			@require_torch
			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Optional[int]:
						import torch

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.PYTORCH    )      ,			torch.equal    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.PYTORCH    )      ,			torch.equal    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      int:
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""Python Tokenizer"""    ):
									self.assertFalse(tokenizer_p("""Small example to_encode"""    ).is_fast    )

						with self.subTest("""Rust Tokenizer"""    ):
									self.assertTrue(tokenizer_r("""Small example to_encode"""    ).is_fast    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= tokenizer_r(["""Test""", """\xad""", """test"""]      ,			is_split_into_words=__UpperCAmelCase    )

						self.assertEqual(encoded.word_to_tokens(0    )      ,			TokenSpan(start=1      ,			end=2    )    )
						self.assertEqual(encoded.word_to_tokens(1    )      ,			__UpperCAmelCase    )
						self.assertEqual(encoded.word_to_tokens(2    )      ,			TokenSpan(start=2      ,			end=3    )    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_torch
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_tf
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      int:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_flax
			def    SCREAMING_SNAKE_CASE			(   self			:  int    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": np.array([0, 1, 2]    )}, {"""input_ids""": np.array([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			np.ndarray    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""np"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			np.ndarray    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_torch
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Optional[int]:
						import torch

						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": torch.tensor([0, 1, 2]    )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			torch.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""pt"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			torch.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_tf
			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      Dict:
						import tensorflow as tf

						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": tf.constant([0, 1, 2]    )}, {"""input_ids""": tf.constant([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			tf.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""tf"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			tf.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
						PreTrainedTokenizerFast(tokenizer_object=__UpperCAmelCase    )





			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  str    )				->      int:
						SCREAMING_SNAKE_CASE__									= Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
						with tempfile.TemporaryDirectory() as tmpdirname:
									bert_tokenizer.save(os.path.join(__UpperCAmelCase      ,			"""tokenizer.json"""    )    )
									PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCAmelCase      ,			"""tokenizer.json"""    )    )


