





import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


UpperCamelCase_							=     logging.get_logger(__name__)




@dataclass
class 					UpperCamelCase_    :
							lowerCamelCase_          =   42
							lowerCamelCase_          =   42
							lowerCamelCase_          =   None
							lowerCamelCase_          =   None


							def        _snake_case					(							self      :Tuple						)				->				Optional[Any]:



									"""simple docstring"""



									return json.dumps(dataclasses.asdict(self						)							,			indent=2						) + "\n"




@dataclass(frozen=SCREAMING_SNAKE_CASE__      )
class 					UpperCamelCase_    :
							lowerCamelCase_          =   42
							lowerCamelCase_          =   None
							lowerCamelCase_          =   None
							lowerCamelCase_          =   None


							def        _snake_case					(							self      :Any						)				->				Optional[int]:



									"""simple docstring"""



									return json.dumps(dataclasses.asdict(self						)						) + "\n"




class 					UpperCamelCase_    :


							def        _snake_case					(							self      :Any							,			__A      :Optional[Any]						)				->				int:



									"""simple docstring"""



									raise NotImplementedError()


							def        _snake_case					(							self      :Dict							,			__A      :Union[str, Any]						)				->				Tuple:



									"""simple docstring"""



									raise NotImplementedError()


							def        _snake_case					(							self      :Any							,			__A      :Optional[int]						)				->				Any:



									"""simple docstring"""



									raise NotImplementedError()


							def        _snake_case					(							self      :Optional[int]							,			__A      :Tuple						)				->				List[str]:



									"""simple docstring"""



									raise NotImplementedError()


							def        _snake_case					(							self      :Any						)				->				Optional[int]:



									"""simple docstring"""



									raise NotImplementedError()


							def        _snake_case					(							self      :Tuple							,			__A      :Union[str, Any]						)				->				int:



									"""simple docstring"""



									if len(self.get_labels()						) > 1:
											SCREAMING_SNAKE_CASE__								= self.get_labels()[int(example.label						)]
									return example


							@classmethod
							def        _snake_case					(							cls      :int							,			__A      :Dict							,			__A      :str=None						)				->				int:



									"""simple docstring"""



									with open(__A							,			"""r"""							,			encoding="""utf-8-sig"""						) as f:
											return list(csv.reader(__A							,			delimiter="""\t"""							,			quotechar=__A						)						)




class 					UpperCamelCase_    (       SCREAMING_SNAKE_CASE__      ):


							def __init__(							self      :Union[str, Any]							,			__A      :Tuple=None							,			__A      :List[Any]=None							,			__A      :Dict="classification"							,			__A      :Union[str, Any]=False						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [] if labels is None else labels
									SCREAMING_SNAKE_CASE__								= [] if examples is None else examples
									SCREAMING_SNAKE_CASE__								= mode
									SCREAMING_SNAKE_CASE__								= verbose


							def __len__(							self      :Dict						)				->				Any:



									"""simple docstring"""



									return len(self.examples						)


							def __getitem__(							self      :Any							,			__A      :Dict						)				->				Optional[int]:



									"""simple docstring"""



									if isinstance(__A							,			__A						):
											return SingleSentenceClassificationProcessor(labels=self.labels							,			examples=self.examples[idx]						)
									return self.examples[idx]


							@classmethod
							def        _snake_case					(							cls      :Optional[int]							,			__A      :Optional[int]							,			__A      :Optional[Any]=""							,			__A      :Optional[int]=0							,			__A      :List[Any]=1							,			__A      :List[str]=None							,			__A      :str=False							,			**__A      :List[str]						)				->				Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= cls(**__A						)
									processor.add_examples_from_csv(
									    __A							,			split_name=__A							,			column_label=__A							,			column_text=__A							,			column_id=__A							,			skip_first_row=__A							,			overwrite_labels=__A							,			overwrite_examples=__A							,			)
									return processor


							@classmethod
							def        _snake_case					(							cls      :List[Any]							,			__A      :Union[str, Any]							,			__A      :Optional[Any]=None							,			**__A      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= cls(**__A						)
									processor.add_examples(__A							,			labels=__A						)
									return processor


							def        _snake_case					(							self      :str							,			__A      :Union[str, Any]							,			__A      :str=""							,			__A      :Union[str, Any]=0							,			__A      :Optional[int]=1							,			__A      :Dict=None							,			__A      :Optional[Any]=False							,			__A      :Union[str, Any]=False							,			__A      :Optional[int]=False							,			)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self._read_tsv(__A						)
									if skip_first_row:
											SCREAMING_SNAKE_CASE__								= lines[1:]
									SCREAMING_SNAKE_CASE__								= []
									SCREAMING_SNAKE_CASE__								= []
									SCREAMING_SNAKE_CASE__								= []
									for i, line in enumerate(__A						):
											texts.append(line[column_text]						)
											labels.append(line[column_label]						)
											if column_id is not None:
													ids.append(line[column_id]						)
											else:
													SCREAMING_SNAKE_CASE__								= f'''{split_name}-{i}''' if split_name else str(__A						)
													ids.append(__A						)

									return self.add_examples(
									    __A							,			__A							,			__A							,			overwrite_labels=__A							,			overwrite_examples=__A						)


							def        _snake_case					(							self      :int							,			__A      :Optional[Any]							,			__A      :Optional[int]=None							,			__A      :int=None							,			__A      :Optional[Any]=False							,			__A      :List[str]=False						)				->				Union[str, Any]:



									"""simple docstring"""



									if labels is not None and len(__A						) != len(__A						):
											raise ValueError(
											    f'''Text and labels have mismatched lengths {len(__A						)} and {len(__A						)}'''						)
									if ids is not None and len(__A						) != len(__A						):
											raise ValueError(f'''Text and ids have mismatched lengths {len(__A						)} and {len(__A						)}'''						)
									if ids is None:
											SCREAMING_SNAKE_CASE__								= [None] * len(__A						)
									if labels is None:
											SCREAMING_SNAKE_CASE__								= [None] * len(__A						)
									SCREAMING_SNAKE_CASE__								= []
									SCREAMING_SNAKE_CASE__								= set()
									for text_or_text_and_label, label, guid in zip(__A							,			__A							,			__A						):
											if isinstance(__A							,			(tuple, list)						) and label is None:
													SCREAMING_SNAKE_CASE__		,  SCREAMING_SNAKE_CASE__								= text_or_text_and_label
											else:
													SCREAMING_SNAKE_CASE__								= text_or_text_and_label
											added_labels.add(__A						)
											examples.append(InputExample(guid=__A							,			text_a=__A							,			text_b=__A							,			label=__A						)						)

									# Update examples
									if overwrite_examples:
											SCREAMING_SNAKE_CASE__								= examples
									else:
											self.examples.extend(__A						)

									# Update labels
									if overwrite_labels:
											SCREAMING_SNAKE_CASE__								= list(__A						)
									else:
											SCREAMING_SNAKE_CASE__								= list(set(self.labels						).union(__A						)						)

									return self.examples


							def        _snake_case					(							self      :int							,			__A      :Optional[int]							,			__A      :Union[str, Any]=None							,			__A      :List[str]=False							,			__A      :Optional[Any]=0							,			__A      :List[str]=True							,			__A      :str=None							,			)				->				Any:



									"""simple docstring"""



									if max_length is None:
											SCREAMING_SNAKE_CASE__								= tokenizer.max_len

									SCREAMING_SNAKE_CASE__								= {label: i for i, label in enumerate(self.labels						)}

									SCREAMING_SNAKE_CASE__								= []
									for ex_index, example in enumerate(self.examples						):
											if ex_index % 1_0000 == 0:
													logger.info(f'''Tokenizing example {ex_index}'''						)

											SCREAMING_SNAKE_CASE__								= tokenizer.encode(
											    example.text_a							,			add_special_tokens=__A							,			max_length=min(__A							,			tokenizer.max_len						)							,			)
											all_input_ids.append(__A						)

									SCREAMING_SNAKE_CASE__								= max(len(__A						) for input_ids in all_input_ids						)

									SCREAMING_SNAKE_CASE__								= []
									for ex_index, (input_ids, example) in enumerate(zip(__A							,			self.examples						)						):
											if ex_index % 1_0000 == 0:
													logger.info(f'''Writing example {ex_index}/{len(self.examples						)}'''						)
											# The mask has 1 for real tokens and 0 for padding tokens. Only real
											# tokens are attended to.
											SCREAMING_SNAKE_CASE__								= [1 if mask_padding_with_zero else 0] * len(__A						)

											# Zero-pad up to the sequence length.
											SCREAMING_SNAKE_CASE__								= batch_length - len(__A						)
											if pad_on_left:
													SCREAMING_SNAKE_CASE__								= ([pad_token] * padding_length) + input_ids
													SCREAMING_SNAKE_CASE__								= ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
											else:
													SCREAMING_SNAKE_CASE__								= input_ids + ([pad_token] * padding_length)
													SCREAMING_SNAKE_CASE__								= attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

											if len(__A						) != batch_length:
													raise ValueError(f'''Error with input length {len(__A						)} vs {batch_length}'''						)
											if len(__A						) != batch_length:
													raise ValueError(f'''Error with input length {len(__A						)} vs {batch_length}'''						)

											if self.mode == "classification":
													SCREAMING_SNAKE_CASE__								= label_map[example.label]
											elif self.mode == "regression":
													SCREAMING_SNAKE_CASE__								= float(example.label						)
											else:
													raise ValueError(self.mode						)

											if ex_index < 5 and self.verbose:
													logger.info("""*** Example ***"""						)
													logger.info(f'''guid: {example.guid}'''						)
													logger.info(f'''input_ids: {' '.join([str(__A						) for x in input_ids]						)}'''						)
													logger.info(f'''attention_mask: {' '.join([str(__A						) for x in attention_mask]						)}'''						)
													logger.info(f'''label: {example.label} (id = {label})'''						)

											features.append(InputFeatures(input_ids=__A							,			attention_mask=__A							,			label=__A						)						)

									if return_tensors is None:
											return features
									elif return_tensors == "tf":
											if not is_tf_available():
													raise RuntimeError("""return_tensors set to 'tf' but TensorFlow 2.0 can't be imported"""						)
											import tensorflow as tf

											def gen():
													for ex in features:
															yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

											SCREAMING_SNAKE_CASE__								= tf.data.Dataset.from_generator(
											    __A							,			({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa}, tf.intaa)							,			({"""input_ids""": tf.TensorShape([None]						), """attention_mask""": tf.TensorShape([None]						)}, tf.TensorShape([]						))							,			)
											return dataset
									elif return_tensors == "pt":
											if not is_torch_available():
													raise RuntimeError("""return_tensors set to 'pt' but PyTorch can't be imported"""						)
											import torch
											from torch.utils.data import TensorDataset

											SCREAMING_SNAKE_CASE__								= torch.tensor([f.input_ids for f in features]							,			dtype=torch.long						)
											SCREAMING_SNAKE_CASE__								= torch.tensor([f.attention_mask for f in features]							,			dtype=torch.long						)
											if self.mode == "classification":
													SCREAMING_SNAKE_CASE__								= torch.tensor([f.label for f in features]							,			dtype=torch.long						)
											elif self.mode == "regression":
													SCREAMING_SNAKE_CASE__								= torch.tensor([f.label for f in features]							,			dtype=torch.float						)

											SCREAMING_SNAKE_CASE__								= TensorDataset(__A							,			__A							,			__A						)
											return dataset
									else:
											raise ValueError("""return_tensors should be one of 'tf' or 'pt'"""						)






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					UpperCamelCase_    (       unittest.TestCase      ):


							def        _snake_case					(							self      :str							,			__A      :str						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= list(tokenizer_class.max_model_input_sizes.keys()						)
									for model_name in sa_models[:1]:
											SCREAMING_SNAKE_CASE__								= tokenizer_class.from_pretrained(__A						)
											self.assertIsNotNone(__A						)
											self.assertIsInstance(__A							,			__A						)
											self.assertIsInstance(__A							,			__A						)

											for special_tok in tokenizer.all_special_tokens:
													self.assertIsInstance(__A							,			__A						)
													SCREAMING_SNAKE_CASE__								= tokenizer.convert_tokens_to_ids(__A						)
													self.assertIsInstance(__A							,			__A						)


							def        _snake_case					(							self      :List[str]							,			__A      :BatchEncoding							,			__A      :Optional[Callable] = None						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= pickle.dumps(__A						)
									self.assertIsNotNone(__A						)

									SCREAMING_SNAKE_CASE__								= pickle.loads(__A						)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast							,			be_original.is_fast						)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
											self.assertIsNotNone(be_restored.encodings						)
									else:
											self.assertIsNone(be_restored.encodings						)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()							,			be_restored.values()						):
											if equal_op:
													self.assertTrue(equal_op(__A							,			__A						)						)
											else:
													self.assertEqual(__A							,			__A						)


							@slow
							def        _snake_case					(							self      :Dict						)				->				Optional[int]:



									"""simple docstring"""



									self.check_tokenizer_from_pretrained(__A						)


							def        _snake_case					(							self      :List[Any]						)				->				Dict:



									"""simple docstring"""



									self.assertEqual(TensorType("""tf"""						)							,			TensorType.TENSORFLOW						)
									self.assertEqual(TensorType("""pt"""						)							,			TensorType.PYTORCH						)
									self.assertEqual(TensorType("""np"""						)							,			TensorType.NUMPY						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				List[Any]:



									"""simple docstring"""



									import numpy as np

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)


							@require_tf
							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				Optional[Any]:



									"""simple docstring"""



									import tensorflow as tf

									def tf_array_equals(__A      :Union[str, Any]							,			__A      :Any						):
											return tf.reduce_all(tf.equal(__A							,			__A						)						)

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)


							@require_torch
							@require_tokenizers
							def        _snake_case					(							self      :List[Any]						)				->				int:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""Python Tokenizer"""						):
											self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

									with self.subTest("""Rust Tokenizer"""						):
											self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)


							@require_tokenizers
							def        _snake_case					(							self      :List[str]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= tokenizer_r(["""Test""", """\xad""", """test"""]							,			is_split_into_words=__A						)

									self.assertEqual(encoded.word_to_tokens(0						)							,			TokenSpan(start=1							,			end=2						)						)
									self.assertEqual(encoded.word_to_tokens(1						)							,			__A						)
									self.assertEqual(encoded.word_to_tokens(2						)							,			TokenSpan(start=2							,			end=3						)						)


							def        _snake_case					(							self      :Tuple						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_tf
							def        _snake_case					(							self      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_flax
							def        _snake_case					(							self      :Dict						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							def        _snake_case					(							self      :List[Any]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""np"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				Dict:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""pt"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tf
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									import tensorflow as tf

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""tf"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tokenizers
							def        _snake_case					(							self      :Tuple						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									PreTrainedTokenizerFast(tokenizer_object=__A						)


							@require_tokenizers
							def        _snake_case					(							self      :Any						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									with tempfile.TemporaryDirectory() as tmpdirname:
											bert_tokenizer.save(os.path.join(__A							,			"""tokenizer.json"""						)						)
											PreTrainedTokenizerFast(tokenizer_file=os.path.join(__A							,			"""tokenizer.json"""						)						)