





import enum
import inspect
from typing import Iterable, List, Optional, Tuple, Union




class 					UpperCamelCase_    (       enum.Enum      ):
							lowerCamelCase_          =   "timm"
							lowerCamelCase_          =   "transformers"






def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							Optional[Iterable[str]]   ,				UpperCamelCase__:							Optional[Iterable[int]]   ,				UpperCamelCase__:							Optional[Iterable[str]]      ):
		if stage_names is None:
				raise ValueError("""Stage_names must be set for transformers backbones"""      )

		if out_features is not None:
				if not isinstance(UpperCamelCase__   ,				(list,)      ):
						raise ValueError(f'''out_features must be a list {type(UpperCamelCase__      )}'''      )
				if any(feat not in stage_names for feat in out_features      ):
						raise ValueError(f'''out_features must be a subset of stage_names: {stage_names} got {out_features}'''      )

		if out_indices is not None:
				if not isinstance(UpperCamelCase__   ,				(list, tuple)      ):
						raise ValueError(f'''out_indices must be a list or tuple, got {type(UpperCamelCase__      )}'''      )
				if any(idx >= len(UpperCamelCase__      ) for idx in out_indices      ):
						raise ValueError("""out_indices must be valid indices for stage_names {stage_names}, got {out_indices}"""      )

		if out_features is not None and out_indices is not None:
				if len(UpperCamelCase__      ) != len(UpperCamelCase__      ):
						raise ValueError("""out_features and out_indices should have the same length if both are set"""      )
				if out_features != [stage_names[idx] for idx in out_indices]:
						raise ValueError("""out_features and out_indices should correspond to the same stages if both are set"""      )






def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							Optional[List[str]]   ,				UpperCamelCase__:							Optional[Union[List[int], Tuple[int]]]   ,				UpperCamelCase__:							List[str]   ,				):
		if out_indices is None and out_features is None:
				SCREAMING_SNAKE_CASE__								= [len(UpperCamelCase__      ) - 1]
				SCREAMING_SNAKE_CASE__								= [stage_names[-1]]
		elif out_indices is None and out_features is not None:
				SCREAMING_SNAKE_CASE__								= [stage_names.index(UpperCamelCase__      ) for layer in out_features]
		elif out_features is None and out_indices is not None:
				SCREAMING_SNAKE_CASE__								= [stage_names[idx] for idx in out_indices]
		return out_features, out_indices






def 			SCREAMING_SNAKE_CASE__							(    UpperCamelCase__:							Optional[List[str]]   ,				UpperCamelCase__:							Optional[Union[List[int], Tuple[int]]]   ,				UpperCamelCase__:							List[str]   ,				):
		verify_out_features_out_indices(out_features=UpperCamelCase__   ,				out_indices=UpperCamelCase__   ,				stage_names=UpperCamelCase__      )
		SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= _align_output_features_output_indices(
		    out_features=UpperCamelCase__   ,				out_indices=UpperCamelCase__   ,				stage_names=UpperCamelCase__      )
		# Verify that the aligned out_features and out_indices are valid
		verify_out_features_out_indices(out_features=UpperCamelCase__   ,				out_indices=UpperCamelCase__   ,				stage_names=UpperCamelCase__      )
		return output_features, output_indices




class 					UpperCamelCase_    :
							lowerCamelCase_          =   None


							def        _snake_case					(							self      :Optional[Any]							,			__A      :Dict						)				->				None:



									"""simple docstring"""



									if getattr(self							,			"""_backbone"""							,			__A						) is None:
											raise ValueError("""self._backbone must be set before calling _init_timm_backbone"""						)

									# These will diagree with the defaults for the transformers models e.g. for resnet50
									# the transformer model has out_features = ['stem', 'stage1', 'stage2', 'stage3', 'stage4']
									# the timm model has out_features = ['act', 'layer1', 'layer2', 'layer3', 'layer4']
									SCREAMING_SNAKE_CASE__								= [stage["""module"""] for stage in self._backbone.feature_info.info]
									SCREAMING_SNAKE_CASE__								= [stage["""num_chs"""] for stage in self._backbone.feature_info.info]
									SCREAMING_SNAKE_CASE__								= self._backbone.feature_info.out_indices
									SCREAMING_SNAKE_CASE__								= self._backbone.feature_info.module_name()

									# We verify the out indices and out features are valid
									verify_out_features_out_indices(
									    out_features=__A							,			out_indices=__A							,			stage_names=self.stage_names						)
									SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= out_features, out_indices


							def        _snake_case					(							self      :Optional[Any]							,			__A      :Tuple						)				->				None:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= getattr(__A							,			"""stage_names"""						)
									SCREAMING_SNAKE_CASE__								= getattr(__A							,			"""out_features"""							,			__A						)
									SCREAMING_SNAKE_CASE__								= getattr(__A							,			"""out_indices"""							,			__A						)

									SCREAMING_SNAKE_CASE__								= stage_names
									SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= get_aligned_output_features_output_indices(
									    out_features=__A							,			out_indices=__A							,			stage_names=__A						)
									# Number of channels for each stage. This is set in the transformer backbone model init
									SCREAMING_SNAKE_CASE__								= None


							def        _snake_case					(							self      :List[str]							,			__A      :List[Any]						)				->				None:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= config

									SCREAMING_SNAKE_CASE__								= getattr(__A							,			"""use_timm_backbone"""							,			__A						)
									SCREAMING_SNAKE_CASE__								= BackboneType.TIMM if self.use_timm_backbone else BackboneType.TRANSFORMERS

									if self.backbone_type == BackboneType.TIMM:
											self._init_timm_backbone(__A						)
									elif self.backbone_type == BackboneType.TRANSFORMERS:
											self._init_transformers_backbone(__A						)
									else:
											raise ValueError(f'''backbone_type {self.backbone_type} not supported.'''						)


							@property
							def        _snake_case					(							self      :Optional[int]						)				->				Any:



									"""simple docstring"""



									return self._out_features


							@out_features.setter
							def        _snake_case					(							self      :Union[str, Any]							,			__A      :List[str]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= get_aligned_output_features_output_indices(
									    out_features=__A							,			out_indices=__A							,			stage_names=self.stage_names						)


							@property
							def        _snake_case					(							self      :Optional[int]						)				->				str:



									"""simple docstring"""



									return self._out_indices


							@out_indices.setter
							def        _snake_case					(							self      :int							,			__A      :Union[Tuple[int], List[int]]						)				->				Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= get_aligned_output_features_output_indices(
									    out_features=__A							,			out_indices=__A							,			stage_names=self.stage_names						)


							@property
							def        _snake_case					(							self      :str						)				->				Optional[Any]:



									"""simple docstring"""



									return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names						)}


							@property
							def        _snake_case					(							self      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									return [self.out_feature_channels[name] for name in self.out_features]


							def        _snake_case					(							self      :str							,			*__A      :int							,			**__A      :Optional[int]						)				->				List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= dict(inspect.signature(self.forward						).parameters						)
									SCREAMING_SNAKE_CASE__								= {k: v for k, v in kwargs.items() if k in signature}
									return self(*__A							,			**__A						)


							def        _snake_case					(							self      :str							,			__A      :Optional[Any]							,			__A      :Optional[bool] = None							,			__A      :Optional[bool] = None							,			__A      :Optional[bool] = None							,			)				->				Optional[int]:



									"""simple docstring"""



									raise NotImplementedError("""This method should be implemented by the derived class."""						)


							def        _snake_case					(							self      :int						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= super().to_dict()
									SCREAMING_SNAKE_CASE__								= output.pop("""_out_features"""						)
									SCREAMING_SNAKE_CASE__								= output.pop("""_out_indices"""						)
									return output




class 					UpperCamelCase_    :


							@property
							def        _snake_case					(							self      :str						)				->				List[Any]:



									"""simple docstring"""



									return self._out_features


							@out_features.setter
							def        _snake_case					(							self      :Union[str, Any]							,			__A      :List[str]						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= get_aligned_output_features_output_indices(
									    out_features=__A							,			out_indices=__A							,			stage_names=self.stage_names						)


							@property
							def        _snake_case					(							self      :Union[str, Any]						)				->				List[str]:



									"""simple docstring"""



									return self._out_indices


							@out_indices.setter
							def        _snake_case					(							self      :Optional[int]							,			__A      :Union[Tuple[int], List[int]]						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= get_aligned_output_features_output_indices(
									    out_features=__A							,			out_indices=__A							,			stage_names=self.stage_names						)


							def        _snake_case					(							self      :int						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= super().to_dict()
									SCREAMING_SNAKE_CASE__								= output.pop("""_out_features"""						)
									SCREAMING_SNAKE_CASE__								= output.pop("""_out_indices"""						)
									return output






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					UpperCamelCase_    (       unittest.TestCase      ):


							def        _snake_case					(							self      :str							,			__A      :str						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= list(tokenizer_class.max_model_input_sizes.keys()						)
									for model_name in sa_models[:1]:
											SCREAMING_SNAKE_CASE__								= tokenizer_class.from_pretrained(__A						)
											self.assertIsNotNone(__A						)
											self.assertIsInstance(__A							,			__A						)
											self.assertIsInstance(__A							,			__A						)

											for special_tok in tokenizer.all_special_tokens:
													self.assertIsInstance(__A							,			__A						)
													SCREAMING_SNAKE_CASE__								= tokenizer.convert_tokens_to_ids(__A						)
													self.assertIsInstance(__A							,			__A						)


							def        _snake_case					(							self      :List[str]							,			__A      :BatchEncoding							,			__A      :Optional[Callable] = None						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= pickle.dumps(__A						)
									self.assertIsNotNone(__A						)

									SCREAMING_SNAKE_CASE__								= pickle.loads(__A						)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast							,			be_original.is_fast						)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
											self.assertIsNotNone(be_restored.encodings						)
									else:
											self.assertIsNone(be_restored.encodings						)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()							,			be_restored.values()						):
											if equal_op:
													self.assertTrue(equal_op(__A							,			__A						)						)
											else:
													self.assertEqual(__A							,			__A						)


							@slow
							def        _snake_case					(							self      :Dict						)				->				Optional[int]:



									"""simple docstring"""



									self.check_tokenizer_from_pretrained(__A						)


							def        _snake_case					(							self      :List[Any]						)				->				Dict:



									"""simple docstring"""



									self.assertEqual(TensorType("""tf"""						)							,			TensorType.TENSORFLOW						)
									self.assertEqual(TensorType("""pt"""						)							,			TensorType.PYTORCH						)
									self.assertEqual(TensorType("""np"""						)							,			TensorType.NUMPY						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				List[Any]:



									"""simple docstring"""



									import numpy as np

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)


							@require_tf
							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				Optional[Any]:



									"""simple docstring"""



									import tensorflow as tf

									def tf_array_equals(__A      :Union[str, Any]							,			__A      :Any						):
											return tf.reduce_all(tf.equal(__A							,			__A						)						)

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)


							@require_torch
							@require_tokenizers
							def        _snake_case					(							self      :List[Any]						)				->				int:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""Python Tokenizer"""						):
											self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

									with self.subTest("""Rust Tokenizer"""						):
											self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)


							@require_tokenizers
							def        _snake_case					(							self      :List[str]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= tokenizer_r(["""Test""", """\xad""", """test"""]							,			is_split_into_words=__A						)

									self.assertEqual(encoded.word_to_tokens(0						)							,			TokenSpan(start=1							,			end=2						)						)
									self.assertEqual(encoded.word_to_tokens(1						)							,			__A						)
									self.assertEqual(encoded.word_to_tokens(2						)							,			TokenSpan(start=2							,			end=3						)						)


							def        _snake_case					(							self      :Tuple						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_tf
							def        _snake_case					(							self      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_flax
							def        _snake_case					(							self      :Dict						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							def        _snake_case					(							self      :List[Any]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""np"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				Dict:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""pt"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tf
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									import tensorflow as tf

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""tf"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tokenizers
							def        _snake_case					(							self      :Tuple						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									PreTrainedTokenizerFast(tokenizer_object=__A						)


							@require_tokenizers
							def        _snake_case					(							self      :Any						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									with tempfile.TemporaryDirectory() as tmpdirname:
											bert_tokenizer.save(os.path.join(__A							,			"""tokenizer.json"""						)						)
											PreTrainedTokenizerFast(tokenizer_file=os.path.join(__A							,			"""tokenizer.json"""						)						)