





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 					UpperCamelCase_    (       _lowerCAmelCase			,					unittest.TestCase      ):
							lowerCamelCase_          =   ElectraTokenizer
							lowerCamelCase_          =   ElectraTokenizerFast
							lowerCamelCase_          =   True
							lowerCamelCase_          =   True
							lowerCamelCase_          =   filter_non_english


							def        _snake_case					(							self      :Optional[Any]						)				->				int:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__								= [
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									SCREAMING_SNAKE_CASE__								= os.path.join(self.tmpdirname							,			VOCAB_FILES_NAMES["""vocab_file"""]						)
									with open(self.vocab_file							,			"""w"""							,			encoding="""utf-8"""						) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)


							def        _snake_case					(							self      :Tuple							,			__A      :int						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= """UNwant\u00E9d,running"""
									SCREAMING_SNAKE_CASE__								= """unwanted, running"""
									return input_text, output_text


							def        _snake_case					(							self      :List[str]						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.tokenizer_class(self.vocab_file						)

									SCREAMING_SNAKE_CASE__								= tokenizer.tokenize("""UNwant\u00E9d,running"""						)
									self.assertListEqual(_lowerCamelCase							,			["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase						)							,			[9, 6, 7, 12, 10, 11]						)


							def        _snake_case					(							self      :List[Any]						)				->				Union[str, Any]:



									"""simple docstring"""



									if not self.test_rust_tokenizer:
											return

									SCREAMING_SNAKE_CASE__								= self.get_tokenizer()
									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer()

									SCREAMING_SNAKE_CASE__								= """UNwant\u00E9d,running"""

									SCREAMING_SNAKE_CASE__								= tokenizer.tokenize(_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.tokenize(_lowerCamelCase						)
									self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)

									SCREAMING_SNAKE_CASE__								= tokenizer.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)
									self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)

									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE__								= tokenizer.encode(_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.encode(_lowerCamelCase						)
									self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)

									# With lower casing
									SCREAMING_SNAKE_CASE__								= self.get_tokenizer(do_lower_case=_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer(do_lower_case=_lowerCamelCase						)

									SCREAMING_SNAKE_CASE__								= """UNwant\u00E9d,running"""

									SCREAMING_SNAKE_CASE__								= tokenizer.tokenize(_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.tokenize(_lowerCamelCase						)
									self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)

									SCREAMING_SNAKE_CASE__								= tokenizer.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)
									self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)

									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE__								= tokenizer.encode(_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= rust_tokenizer.encode(_lowerCamelCase						)
									self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)


							def        _snake_case					(							self      :Optional[Any]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""						)							,			["""ah""", """\u535A""", """\u63A8""", """zz"""]						)


							def        _snake_case					(							self      :int						)				->				List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)							,			["""hello""", """!""", """how""", """are""", """you""", """?"""]						)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)							,			["""hello"""]						)


							def        _snake_case					(							self      :List[str]						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase							,			strip_accents=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)							,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]						)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)							,			["""h\u00E9llo"""]						)


							def        _snake_case					(							self      :Union[str, Any]						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase							,			strip_accents=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)							,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)							,			["""hello"""]						)


							def        _snake_case					(							self      :int						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)							,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)							,			["""hello"""]						)


							def        _snake_case					(							self      :Tuple						)				->				List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)							,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)


							def        _snake_case					(							self      :Union[str, Any]						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase							,			strip_accents=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)							,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)


							def        _snake_case					(							self      :Union[str, Any]						)				->				List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase							,			strip_accents=_lowerCamelCase						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)							,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)


							def        _snake_case					(							self      :Dict						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BasicTokenizer(do_lower_case=_lowerCamelCase							,			never_split=["""[UNK]"""]						)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""						)							,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]						)


							def        _snake_case					(							self      :int						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									SCREAMING_SNAKE_CASE__								= {}
									for i, token in enumerate(_lowerCamelCase						):
											SCREAMING_SNAKE_CASE__								= i
									SCREAMING_SNAKE_CASE__								= WordpieceTokenizer(vocab=_lowerCamelCase							,			unk_token="""[UNK]"""						)

									self.assertListEqual(tokenizer.tokenize(""""""						)							,			[]						)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""						)							,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]						)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""						)							,			["""[UNK]""", """runn""", """##ing"""]						)


							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									self.assertTrue(_is_whitespace(""" """						)						)
									self.assertTrue(_is_whitespace("""\t"""						)						)
									self.assertTrue(_is_whitespace("""\r"""						)						)
									self.assertTrue(_is_whitespace("""\n"""						)						)
									self.assertTrue(_is_whitespace("""\u00A0"""						)						)

									self.assertFalse(_is_whitespace("""A"""						)						)
									self.assertFalse(_is_whitespace("""-"""						)						)


							def        _snake_case					(							self      :Any						)				->				Dict:



									"""simple docstring"""



									self.assertTrue(_is_control("""\u0005"""						)						)

									self.assertFalse(_is_control("""A"""						)						)
									self.assertFalse(_is_control(""" """						)						)
									self.assertFalse(_is_control("""\t"""						)						)
									self.assertFalse(_is_control("""\r"""						)						)


							def        _snake_case					(							self      :Optional[int]						)				->				Optional[int]:



									"""simple docstring"""



									self.assertTrue(_is_punctuation("""-"""						)						)
									self.assertTrue(_is_punctuation("""$"""						)						)
									self.assertTrue(_is_punctuation("""`"""						)						)
									self.assertTrue(_is_punctuation("""."""						)						)

									self.assertFalse(_is_punctuation("""A"""						)						)
									self.assertFalse(_is_punctuation(""" """						)						)


							def        _snake_case					(							self      :Optional[int]						)				->				List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.get_tokenizer()
									SCREAMING_SNAKE_CASE__								= self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(_lowerCamelCase						) for t in ["""Test""", """\xad""", """test"""]]							,			[["""[UNK]"""], [], ["""[UNK]"""]]						)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(_lowerCamelCase						) for t in ["""Test""", """\xad""", """test"""]]							,			[["""[UNK]"""], [], ["""[UNK]"""]]						)


							@slow
							def        _snake_case					(							self      :Optional[int]						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.encode("""sequence builders"""							,			add_special_tokens=_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= tokenizer.encode("""multi-sequence build"""							,			add_special_tokens=_lowerCamelCase						)

									SCREAMING_SNAKE_CASE__								= tokenizer.build_inputs_with_special_tokens(_lowerCamelCase						)
									SCREAMING_SNAKE_CASE__								= tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							,			_lowerCamelCase						)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]


							def        _snake_case					(							self      :Tuple						)				->				str:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													SCREAMING_SNAKE_CASE__								= self.rust_tokenizer_class.from_pretrained(_lowerCamelCase							,			**_lowerCamelCase						)

													SCREAMING_SNAKE_CASE__								= f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE__								= tokenizer_r.encode_plus(
													    _lowerCamelCase							,			return_attention_mask=_lowerCamelCase							,			return_token_type_ids=_lowerCamelCase							,			return_offsets_mapping=_lowerCamelCase							,			add_special_tokens=_lowerCamelCase							,			)

													SCREAMING_SNAKE_CASE__								= tokenizer_r.do_lower_case if hasattr(_lowerCamelCase							,			"""do_lower_case"""						) else False
													SCREAMING_SNAKE_CASE__								= (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """A"""),
													        ((1, 2), ""","""),
													        ((3, 5), """na"""),
													        ((5, 6), """##ï"""),
													        ((6, 8), """##ve"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """Allen"""),
													        ((21, 23), """##NL"""),
													        ((23, 24), """##P"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """a"""),
													        ((1, 2), ""","""),
													        ((3, 8), """naive"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """allen"""),
													        ((21, 23), """##nl"""),
													        ((23, 24), """##p"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]						)						)
													self.assertEqual([e[0] for e in expected_results]							,			tokens["""offset_mapping"""]						)


							def        _snake_case					(							self      :Optional[int]						)				->				Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= ["""的""", """人""", """有"""]
									SCREAMING_SNAKE_CASE__								= """""".join(_lowerCamelCase						)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													SCREAMING_SNAKE_CASE__								= True
													SCREAMING_SNAKE_CASE__								= self.tokenizer_class.from_pretrained(_lowerCamelCase							,			**_lowerCamelCase						)
													SCREAMING_SNAKE_CASE__								= self.rust_tokenizer_class.from_pretrained(_lowerCamelCase							,			**_lowerCamelCase						)

													SCREAMING_SNAKE_CASE__								= tokenizer_p.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)
													SCREAMING_SNAKE_CASE__								= tokenizer_r.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)

													SCREAMING_SNAKE_CASE__								= tokenizer_r.convert_ids_to_tokens(_lowerCamelCase						)
													SCREAMING_SNAKE_CASE__								= tokenizer_p.convert_ids_to_tokens(_lowerCamelCase						)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)
													self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)

													SCREAMING_SNAKE_CASE__								= False
													SCREAMING_SNAKE_CASE__								= self.rust_tokenizer_class.from_pretrained(_lowerCamelCase							,			**_lowerCamelCase						)
													SCREAMING_SNAKE_CASE__								= self.tokenizer_class.from_pretrained(_lowerCamelCase							,			**_lowerCamelCase						)

													SCREAMING_SNAKE_CASE__								= tokenizer_r.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)
													SCREAMING_SNAKE_CASE__								= tokenizer_p.encode(_lowerCamelCase							,			add_special_tokens=_lowerCamelCase						)

													SCREAMING_SNAKE_CASE__								= tokenizer_r.convert_ids_to_tokens(_lowerCamelCase						)
													SCREAMING_SNAKE_CASE__								= tokenizer_p.convert_ids_to_tokens(_lowerCamelCase						)

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE__								= [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase						)
													]
													self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)
													self.assertListEqual(_lowerCamelCase							,			_lowerCamelCase						)






import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					UpperCamelCase_    (       unittest.TestCase      ):


							def        _snake_case					(							self      :str							,			__A      :str						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= list(tokenizer_class.max_model_input_sizes.keys()						)
									for model_name in sa_models[:1]:
											SCREAMING_SNAKE_CASE__								= tokenizer_class.from_pretrained(__A						)
											self.assertIsNotNone(__A						)
											self.assertIsInstance(__A							,			__A						)
											self.assertIsInstance(__A							,			__A						)

											for special_tok in tokenizer.all_special_tokens:
													self.assertIsInstance(__A							,			__A						)
													SCREAMING_SNAKE_CASE__								= tokenizer.convert_tokens_to_ids(__A						)
													self.assertIsInstance(__A							,			__A						)


							def        _snake_case					(							self      :List[str]							,			__A      :BatchEncoding							,			__A      :Optional[Callable] = None						)				->				Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= pickle.dumps(__A						)
									self.assertIsNotNone(__A						)

									SCREAMING_SNAKE_CASE__								= pickle.loads(__A						)

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast							,			be_original.is_fast						)

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
											self.assertIsNotNone(be_restored.encodings						)
									else:
											self.assertIsNone(be_restored.encodings						)

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()							,			be_restored.values()						):
											if equal_op:
													self.assertTrue(equal_op(__A							,			__A						)						)
											else:
													self.assertEqual(__A							,			__A						)


							@slow
							def        _snake_case					(							self      :Dict						)				->				Optional[int]:



									"""simple docstring"""



									self.check_tokenizer_from_pretrained(__A						)


							def        _snake_case					(							self      :List[Any]						)				->				Dict:



									"""simple docstring"""



									self.assertEqual(TensorType("""tf"""						)							,			TensorType.TENSORFLOW						)
									self.assertEqual(TensorType("""pt"""						)							,			TensorType.PYTORCH						)
									self.assertEqual(TensorType("""np"""						)							,			TensorType.NUMPY						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				List[Any]:



									"""simple docstring"""



									import numpy as np

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""						):
											self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""						)						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.NUMPY						)							,			np.array_equal						)


							@require_tf
							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				Optional[Any]:



									"""simple docstring"""



									import tensorflow as tf

									def tf_array_equals(__A      :Union[str, Any]							,			__A      :Any						):
											return tf.reduce_all(tf.equal(__A							,			__A						)						)

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.TENSORFLOW						)							,			__A						)


							@require_torch
							@require_tokenizers
							def        _snake_case					(							self      :List[Any]						)				->				int:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_p("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""						):
											self.assert_dump_and_restore(
											    tokenizer_r("""Small example to encode"""							,			return_tensors=TensorType.PYTORCH						)							,			torch.equal						)


							@require_tokenizers
							def        _snake_case					(							self      :Optional[Any]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)

									with self.subTest("""Python Tokenizer"""						):
											self.assertFalse(tokenizer_p("""Small example to_encode"""						).is_fast						)

									with self.subTest("""Rust Tokenizer"""						):
											self.assertTrue(tokenizer_r("""Small example to_encode"""						).is_fast						)


							@require_tokenizers
							def        _snake_case					(							self      :List[str]						)				->				str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BertTokenizerFast.from_pretrained("""bert-base-cased"""						)
									SCREAMING_SNAKE_CASE__								= tokenizer_r(["""Test""", """\xad""", """test"""]							,			is_split_into_words=__A						)

									self.assertEqual(encoded.word_to_tokens(0						)							,			TokenSpan(start=1							,			end=2						)						)
									self.assertEqual(encoded.word_to_tokens(1						)							,			__A						)
									self.assertEqual(encoded.word_to_tokens(2						)							,			TokenSpan(start=2							,			end=3						)						)


							def        _snake_case					(							self      :Tuple						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""np"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""pt"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_tf
							def        _snake_case					(							self      :Optional[Any]						)				->				Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""tf"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							@require_flax
							def        _snake_case					(							self      :Dict						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(2, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(2,)						)
									# test converting the converted
									with CaptureStderr() as cs:
											SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""						)
									self.assertFalse(len(cs.err						)							,			msg=f'''should have no warning, but got {cs.err}'''						)

									SCREAMING_SNAKE_CASE__								= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}						)
									SCREAMING_SNAKE_CASE__								= batch.convert_to_tensors(tensor_type="""jax"""							,			prepend_batch_axis=__A						)
									self.assertEqual(tensor_batch["""inputs"""].shape							,			(1, 3)						)
									self.assertEqual(tensor_batch["""labels"""].shape							,			(1,)						)


							def        _snake_case					(							self      :List[Any]						)				->				Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": np.array([0, 1, 2]						)}, {"""input_ids""": np.array([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""np"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			np.ndarray						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_torch
							def        _snake_case					(							self      :int						)				->				Dict:



									"""simple docstring"""



									import torch

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": torch.tensor([0, 1, 2]						)}, {"""input_ids""": torch.tensor([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""pt"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			torch.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tf
							def        _snake_case					(							self      :int						)				->				int:



									"""simple docstring"""



									import tensorflow as tf

									SCREAMING_SNAKE_CASE__								= [{"""input_ids""": tf.constant([0, 1, 2]						)}, {"""input_ids""": tf.constant([0, 1, 2, 3]						)}]
									SCREAMING_SNAKE_CASE__								= BertTokenizer.from_pretrained("""bert-base-cased"""						)

									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
									SCREAMING_SNAKE_CASE__								= tokenizer.pad(__A							,			padding=__A							,			return_tensors="""tf"""						)
									self.assertTrue(isinstance(batch["""input_ids"""]							,			tf.Tensor						)						)
									self.assertEqual(batch["""input_ids"""].numpy().tolist()							,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)


							@require_tokenizers
							def        _snake_case					(							self      :Tuple						)				->				Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									PreTrainedTokenizerFast(tokenizer_object=__A						)


							@require_tokenizers
							def        _snake_case					(							self      :Any						)				->				Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								= Tokenizer(WordPiece(unk_token="""[UNK]"""						)						)
									with tempfile.TemporaryDirectory() as tmpdirname:
											bert_tokenizer.save(os.path.join(__A							,			"""tokenizer.json"""						)						)
											PreTrainedTokenizerFast(tokenizer_file=os.path.join(__A							,			"""tokenizer.json"""						)						)