

"""simple docstring"""




import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_							:     Optional[Any]     =	logging.get_logger(__name__)

A_							:     int     =	{
    "src_vocab_file": "vocab-src.json",
    "tgt_vocab_file": "vocab-tgt.json",
    "merges_file": "merges.txt",
}

A_							:     List[str]     =	{
    "src_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"
    },
    "tgt_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"
    },
    "merges_file": {"stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"},
}

A_							:     List[str]     =	{"stas/tiny-wmt19-en-de": 1_024}
A_							:     Tuple     =	{
    "stas/tiny-wmt19-en-de": {
        "langs": ["en", "de"],
        "model_max_length": 1_024,
        "special_tokens_map_file": None,
        "full_tokenizer_file": None,
    }
}

def 		A    (       snake_case__      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= set()
			SCREAMING_SNAKE_CASE__									= word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)      )
						SCREAMING_SNAKE_CASE__									= char
			return pairs

def 		A    (       snake_case__      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= text.replace("""，"""						,			""","""      )
			SCREAMING_SNAKE_CASE__									= re.sub(R"""。\s*"""						,			""". """						,			snake_case__      )
			SCREAMING_SNAKE_CASE__									= text.replace("""、"""						,			""","""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""”"""						,			"""\""""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""“"""						,			"""\""""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""∶"""						,			""":"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""："""						,			""":"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""？"""						,			"""?"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""《"""						,			"""\""""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""》"""						,			"""\""""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""）"""						,			""")"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""！"""						,			"""!"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""（"""						,			"""("""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""；"""						,			""";"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""１"""						,			"""1"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""」"""						,			"""\""""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""「"""						,			"""\""""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""０"""						,			"""0"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""３"""						,			"""3"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""２"""						,			"""2"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""５"""						,			"""5"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""６"""						,			"""6"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""９"""						,			"""9"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""７"""						,			"""7"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""８"""						,			"""8"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""４"""						,			"""4"""      )
			SCREAMING_SNAKE_CASE__									= re.sub(R"""．\s*"""						,			""". """						,			snake_case__      )
			SCREAMING_SNAKE_CASE__									= text.replace("""～"""						,			"""~"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""’"""						,			"""'"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""…"""						,			"""..."""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""━"""						,			"""-"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""〈"""						,			"""<"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""〉"""						,			""">"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""【"""						,			"""["""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""】"""						,			"""]"""      )
			SCREAMING_SNAKE_CASE__									= text.replace("""％"""						,			"""%"""      )
			return text

def 		A    (       snake_case__      ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__									= []
			for char in text:
						SCREAMING_SNAKE_CASE__									= unicodedata.category(snake_case__      )
						if cat.startswith("""C"""      ):
									continue
						output.append(snake_case__      )
			return "".join(snake_case__      )





class 				lowerCamelCase  (A__     ):
			lowerCamelCase__				:						int   						=							VOCAB_FILES_NAMES
			lowerCamelCase__				:						List[Any]   						=							PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__				:						Any   						=							PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase__				:						str   						=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__				:						List[Any]   						=							['input_ids', 'attention_mask']

			def __init__(   self			:  Optional[int]      ,			__UpperCAmelCase			:  Optional[int]=None      ,			__UpperCAmelCase			:  Optional[Any]=None      ,			__UpperCAmelCase			:  List[str]=None      ,			__UpperCAmelCase			:  List[Any]=None      ,			__UpperCAmelCase			:  Any=False      ,			__UpperCAmelCase			:  Tuple="<unk>"      ,			__UpperCAmelCase			:  Optional[int]="<s>"      ,			__UpperCAmelCase			:  Union[str, Any]="</s>"      ,			__UpperCAmelCase			:  Tuple="<pad>"      ,			**__UpperCAmelCase			:  Dict      ,			)				->      Union[str, Any]:
						super().__init__(
						    langs=__UpperCAmelCase      ,			src_vocab_file=__UpperCAmelCase      ,			tgt_vocab_file=__UpperCAmelCase      ,			merges_file=__UpperCAmelCase      ,			do_lower_case=__UpperCAmelCase      ,			unk_token=__UpperCAmelCase      ,			bos_token=__UpperCAmelCase      ,			sep_token=__UpperCAmelCase      ,			pad_token=__UpperCAmelCase      ,			**__UpperCAmelCase      ,			)

						try:
									import sacremoses
						except ImportError:
									raise ImportError(
									    """You need to install sacremoses to use XLMTokenizer. """
									    """See https://pypi.org/project/sacremoses/ for installation."""    )

						SCREAMING_SNAKE_CASE__									= sacremoses

						SCREAMING_SNAKE_CASE__									= src_vocab_file
						SCREAMING_SNAKE_CASE__									= tgt_vocab_file
						SCREAMING_SNAKE_CASE__									= merges_file
						SCREAMING_SNAKE_CASE__									= do_lower_case

						# cache of sm.MosesPunctNormalizer instance
						SCREAMING_SNAKE_CASE__									= {}
						# cache of sm.MosesTokenizer instance
						SCREAMING_SNAKE_CASE__									= {}
						SCREAMING_SNAKE_CASE__									= {}

						if langs and len(__UpperCAmelCase    ) == 2:
									SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__									= langs
						else:
									raise ValueError(
									    F"""arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. """
									    """Usually that means that tokenizer can't find a mapping for the given model path """
									    """in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."""    )

						with open(__UpperCAmelCase      ,			encoding="""utf-8"""    ) as src_vocab_handle:
									SCREAMING_SNAKE_CASE__									= json.load(__UpperCAmelCase    )
						with open(__UpperCAmelCase      ,			encoding="""utf-8"""    ) as tgt_vocab_handle:
									SCREAMING_SNAKE_CASE__									= json.load(__UpperCAmelCase    )
									SCREAMING_SNAKE_CASE__									= {v: k for k, v in tgt_vocab.items()}
						with open(__UpperCAmelCase      ,			encoding="""utf-8"""    ) as merges_handle:
									SCREAMING_SNAKE_CASE__									= merges_handle.read().split("""\n"""    )[:-1]
						SCREAMING_SNAKE_CASE__									= [tuple(merge.split()[:2]    ) for merge in merges]
						SCREAMING_SNAKE_CASE__									= dict(zip(__UpperCAmelCase      ,			range(len(__UpperCAmelCase    )    )    )    )
						SCREAMING_SNAKE_CASE__									= {}

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      Dict[str, int]:
						return self.get_src_vocab()

			@property
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      int:
						return self.src_vocab_size

			def    SCREAMING_SNAKE_CASE			(   self			:  List[str]      ,			__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  List[Any]    )				->      str:
						if lang not in self.cache_moses_punct_normalizer:
									SCREAMING_SNAKE_CASE__									= self.sm.MosesPunctNormalizer(lang=__UpperCAmelCase    )
									SCREAMING_SNAKE_CASE__									= punct_normalizer
						return self.cache_moses_punct_normalizer[lang].normalize(__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Any      ,			__UpperCAmelCase			:  List[Any]      ,			__UpperCAmelCase			:  List[str]    )				->      int:
						if lang not in self.cache_moses_tokenizer:
									SCREAMING_SNAKE_CASE__									= self.sm.MosesTokenizer(lang=__UpperCAmelCase    )
									SCREAMING_SNAKE_CASE__									= moses_tokenizer
						return self.cache_moses_tokenizer[lang].tokenize(
						    __UpperCAmelCase      ,			aggressive_dash_splits=__UpperCAmelCase      ,			return_str=__UpperCAmelCase      ,			escape=__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Union[str, Any]      ,			__UpperCAmelCase			:  Optional[Any]      ,			__UpperCAmelCase			:  Any    )				->      Tuple:
						if lang not in self.cache_moses_detokenizer:
									SCREAMING_SNAKE_CASE__									= self.sm.MosesDetokenizer(lang=__UpperCAmelCase    )
									SCREAMING_SNAKE_CASE__									= moses_detokenizer
						return self.cache_moses_detokenizer[lang].detokenize(__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  int      ,			__UpperCAmelCase			:  List[str]      ,			__UpperCAmelCase			:  Any    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= replace_unicode_punct(__UpperCAmelCase    )
						SCREAMING_SNAKE_CASE__									= self.moses_punct_norm(__UpperCAmelCase      ,			__UpperCAmelCase    )
						SCREAMING_SNAKE_CASE__									= remove_non_printing_char(__UpperCAmelCase    )
						return text

			@property
			def    SCREAMING_SNAKE_CASE			(   self			:  Union[str, Any]    )				->      Union[str, Any]:
						return len(self.encoder    )

			@property
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Union[str, Any]:
						return len(self.decoder    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Union[str, Any]    )				->      Optional[Any]:
						return dict(self.encoder      ,			**self.added_tokens_encoder    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      List[Any]:
						return dict(self.decoder      ,			**self.added_tokens_decoder    )

			def    SCREAMING_SNAKE_CASE			(   self			:  int      ,			__UpperCAmelCase			:  Optional[int]    )				->      Dict:
						SCREAMING_SNAKE_CASE__									= tuple(token[:-1]    ) + (token[-1] + """</w>""",)
						if token in self.cache:
									return self.cache[token]
						SCREAMING_SNAKE_CASE__									= get_pairs(__UpperCAmelCase    )

						if not pairs:
									return token + "</w>"

						while True:
									SCREAMING_SNAKE_CASE__									= min(__UpperCAmelCase      ,			key=lambda __UpperCAmelCase    : self.bpe_ranks.get(__UpperCAmelCase      ,			float("""inf"""    )    )    )
									if bigram not in self.bpe_ranks:
												break
									SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__									= bigram
									SCREAMING_SNAKE_CASE__									= []
									SCREAMING_SNAKE_CASE__									= 0
									while i < len(__UpperCAmelCase    ):
												try:
															SCREAMING_SNAKE_CASE__									= word.index(__UpperCAmelCase      ,			__UpperCAmelCase    )
												except ValueError:
															new_word.extend(word[i:]    )
															break
												else:
															new_word.extend(word[i:j]    )
															SCREAMING_SNAKE_CASE__									= j

												if word[i] == first and i < len(__UpperCAmelCase    ) - 1 and word[i + 1] == second:
															new_word.append(first + second    )
															i += 2
												else:
															new_word.append(word[i]    )
															i += 1
									SCREAMING_SNAKE_CASE__									= tuple(__UpperCAmelCase    )
									SCREAMING_SNAKE_CASE__									= new_word
									if len(__UpperCAmelCase    ) == 1:
												break
									else:
												SCREAMING_SNAKE_CASE__									= get_pairs(__UpperCAmelCase    )
						SCREAMING_SNAKE_CASE__									= """ """.join(__UpperCAmelCase    )
						if word == "\n  </w>":
									SCREAMING_SNAKE_CASE__									= """\n</w>"""
						SCREAMING_SNAKE_CASE__									= word
						return word

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple      ,			__UpperCAmelCase			:  int      ,			__UpperCAmelCase			:  Optional[Any]="en"      ,			__UpperCAmelCase			:  List[str]=False    )				->      int:
						SCREAMING_SNAKE_CASE__									= self.src_lang

						if self.do_lower_case:
									SCREAMING_SNAKE_CASE__									= text.lower()

						if bypass_tokenizer:
									SCREAMING_SNAKE_CASE__									= text.split()
						else:
									SCREAMING_SNAKE_CASE__									= self.moses_pipeline(__UpperCAmelCase      ,			lang=__UpperCAmelCase    )
									SCREAMING_SNAKE_CASE__									= self.moses_tokenize(__UpperCAmelCase      ,			lang=__UpperCAmelCase    )

						SCREAMING_SNAKE_CASE__									= []
						for token in text:
									if token:
												split_tokens.extend(list(self.bpe(__UpperCAmelCase    ).split(""" """    )    )    )

						return split_tokens

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]      ,			__UpperCAmelCase			:  List[Any]    )				->      List[str]:
						return self.encoder.get(__UpperCAmelCase      ,			self.encoder.get(self.unk_token    )    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple      ,			__UpperCAmelCase			:  List[Any]    )				->      int:
						return self.decoder.get(__UpperCAmelCase      ,			self.unk_token    )

			def    SCREAMING_SNAKE_CASE			(   self			:  str      ,			__UpperCAmelCase			:  str    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= [t.replace(""" """      ,			""""""    ).replace("""</w>"""      ,			""" """    ) for t in tokens]
						SCREAMING_SNAKE_CASE__									= """""".join(__UpperCAmelCase    ).split()
						# detokenize
						SCREAMING_SNAKE_CASE__									= self.moses_detokenize(__UpperCAmelCase      ,			self.tgt_lang    )
						return text

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]      ,			__UpperCAmelCase			:  List[int]      ,			__UpperCAmelCase			:  Optional[List[int]] = None    )				->      List[int]:
						SCREAMING_SNAKE_CASE__									= [self.sep_token_id]

						# no bos used in fairseq
						if token_ids_a is None:
									return token_ids_a + sep
						return token_ids_a + sep + token_ids_a + sep

			def    SCREAMING_SNAKE_CASE			(   self			:  Any      ,			__UpperCAmelCase			:  List[int]      ,			__UpperCAmelCase			:  Optional[List[int]] = None      ,			__UpperCAmelCase			:  bool = False    )				->      List[int]:

						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__UpperCAmelCase      ,			token_ids_a=__UpperCAmelCase      ,			already_has_special_tokens=__UpperCAmelCase    )
						# no bos used in fairseq
						if token_ids_a is not None:
									return ([0] * len(__UpperCAmelCase    )) + [1] + ([0] * len(__UpperCAmelCase    )) + [1]
						return ([0] * len(__UpperCAmelCase    )) + [1]

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple      ,			__UpperCAmelCase			:  List[int]      ,			__UpperCAmelCase			:  Optional[List[int]] = None    )				->      List[int]:
						SCREAMING_SNAKE_CASE__									= [self.sep_token_id]

						# no bos used in fairseq
						if token_ids_a is None:
									return len(token_ids_a + sep    ) * [0]
						return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

			def    SCREAMING_SNAKE_CASE			(   self			:  int      ,			__UpperCAmelCase			:  str      ,			__UpperCAmelCase			:  Optional[str] = None    )				->      Tuple[str]:
						if not os.path.isdir(__UpperCAmelCase    ):
									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
									return

						SCREAMING_SNAKE_CASE__									= os.path.join(
						    __UpperCAmelCase      ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""src_vocab_file"""]    )
						SCREAMING_SNAKE_CASE__									= os.path.join(
						    __UpperCAmelCase      ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""tgt_vocab_file"""]    )
						SCREAMING_SNAKE_CASE__									= os.path.join(
						    __UpperCAmelCase      ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]    )

						with open(__UpperCAmelCase      ,			"""w"""      ,			encoding="""utf-8"""    ) as f:
									f.write(json.dumps(self.encoder      ,			indent=2      ,			sort_keys=__UpperCAmelCase      ,			ensure_ascii=__UpperCAmelCase    ) + """\n"""    )

						with open(__UpperCAmelCase      ,			"""w"""      ,			encoding="""utf-8"""    ) as f:
									SCREAMING_SNAKE_CASE__									= {v: k for k, v in self.decoder.items()}
									f.write(json.dumps(__UpperCAmelCase      ,			indent=2      ,			sort_keys=__UpperCAmelCase      ,			ensure_ascii=__UpperCAmelCase    ) + """\n"""    )

						SCREAMING_SNAKE_CASE__									= 0
						with open(__UpperCAmelCase      ,			"""w"""      ,			encoding="""utf-8"""    ) as writer:
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,			key=lambda __UpperCAmelCase    : kv[1]    ):
												if index != token_index:
															logger.warning(
															    F"""Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."""
															    """ Please check that the tokenizer is not corrupted!"""    )
															SCREAMING_SNAKE_CASE__									= token_index
												writer.write(""" """.join(__UpperCAmelCase    ) + """\n"""    )
												index += 1

						return src_vocab_file, tgt_vocab_file, merges_file

			def __getstate__(   self			:  List[Any]    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= self.__dict__.copy()
						SCREAMING_SNAKE_CASE__									= None
						return state





			def __setstate__(   self			:  Optional[Any]      ,			__UpperCAmelCase			:  str    )				->      List[str]:
						SCREAMING_SNAKE_CASE__									= d

						try:
									import sacremoses
						except ImportError:
									raise ImportError(
									    """You need to install sacremoses to use XLMTokenizer. """
									    """See https://pypi.org/project/sacremoses/ for installation."""    )

						SCREAMING_SNAKE_CASE__									= sacremoses





"""simple docstring"""




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece





class 				lowerCamelCase  (unittest.TestCase     ):

			def    SCREAMING_SNAKE_CASE			(   self			:  Union[str, Any]      ,			__UpperCAmelCase			:  str    )				->      int:
						SCREAMING_SNAKE_CASE__									= list(tokenizer_class.max_model_input_sizes.keys()    )
						for model_name in sa_models[:1]:
									SCREAMING_SNAKE_CASE__									= tokenizer_class.from_pretrained(__UpperCAmelCase    )
									self.assertIsNotNone(__UpperCAmelCase    )
									self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )
									self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )

									for special_tok in tokenizer.all_special_tokens:
												self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )
												SCREAMING_SNAKE_CASE__									= tokenizer.convert_tokens_to_ids(__UpperCAmelCase    )
												self.assertIsInstance(__UpperCAmelCase      ,			__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Dict      ,			__UpperCAmelCase			:  BatchEncoding      ,			__UpperCAmelCase			:  Optional[Callable] = None    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= pickle.dumps(__UpperCAmelCase    )
						self.assertIsNotNone(__UpperCAmelCase    )

						SCREAMING_SNAKE_CASE__									= pickle.loads(__UpperCAmelCase    )

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast      ,			be_original.is_fast    )

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
									self.assertIsNotNone(be_restored.encodings    )
						else:
									self.assertIsNone(be_restored.encodings    )

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()      ,			be_restored.values()    ):
									if equal_op:
												self.assertTrue(equal_op(__UpperCAmelCase      ,			__UpperCAmelCase    )    )
									else:
												self.assertEqual(__UpperCAmelCase      ,			__UpperCAmelCase    )

			@slow
			def    SCREAMING_SNAKE_CASE			(   self			:  str    )				->      Optional[Any]:
						self.check_tokenizer_from_pretrained(__UpperCAmelCase    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Tuple    )				->      Optional[int]:
						self.assertEqual(TensorType("""tf"""    )      ,			TensorType.TENSORFLOW    )
						self.assertEqual(TensorType("""pt"""    )      ,			TensorType.PYTORCH    )
						self.assertEqual(TensorType("""np"""    )      ,			TensorType.NUMPY    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      List[str]:
						import numpy as np

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						# Python no tensor
						with self.subTest("""BatchEncoding (Python, return_tensors=None)"""    ):
									self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""    )    )

						with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.NUMPY    )      ,			np.array_equal    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""    ):
									self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""    )    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.NUMPY    )      ,			np.array_equal    )

			@require_tf
			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      int:
						import tensorflow as tf

						def tf_array_equals(__UpperCAmelCase			:  Optional[int]      ,			__UpperCAmelCase			:  str    ):
									return tf.reduce_all(tf.equal(__UpperCAmelCase      ,			__UpperCAmelCase    )    )

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.TENSORFLOW    )      ,			__UpperCAmelCase    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.TENSORFLOW    )      ,			__UpperCAmelCase    )

			@require_torch
			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Optional[int]:
						import torch

						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_p("""Small example to encode"""      ,			return_tensors=TensorType.PYTORCH    )      ,			torch.equal    )

						with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""    ):
									self.assert_dump_and_restore(
									    tokenizer_r("""Small example to encode"""      ,			return_tensors=TensorType.PYTORCH    )      ,			torch.equal    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      int:
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )

						with self.subTest("""Python Tokenizer"""    ):
									self.assertFalse(tokenizer_p("""Small example to_encode"""    ).is_fast    )

						with self.subTest("""Rust Tokenizer"""    ):
									self.assertTrue(tokenizer_r("""Small example to_encode"""    ).is_fast    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[int]    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= BertTokenizerFast.from_pretrained("""bert-base-cased"""    )
						SCREAMING_SNAKE_CASE__									= tokenizer_r(["""Test""", """\xad""", """test"""]      ,			is_split_into_words=__UpperCAmelCase    )

						self.assertEqual(encoded.word_to_tokens(0    )      ,			TokenSpan(start=1      ,			end=2    )    )
						self.assertEqual(encoded.word_to_tokens(1    )      ,			__UpperCAmelCase    )
						self.assertEqual(encoded.word_to_tokens(2    )      ,			TokenSpan(start=2      ,			end=3    )    )

			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""np"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_torch
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Optional[Any]:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""pt"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_tf
			def    SCREAMING_SNAKE_CASE			(   self			:  Any    )				->      int:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""tf"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			@require_flax
			def    SCREAMING_SNAKE_CASE			(   self			:  int    )				->      Tuple:
						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(2, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(2,)    )
						# test converting the converted
						with CaptureStderr() as cs:
									SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""    )
						self.assertFalse(len(cs.err    )      ,			msg=F"""should have no warning, but got {cs.err}"""    )

						SCREAMING_SNAKE_CASE__									= BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}    )
						SCREAMING_SNAKE_CASE__									= batch.convert_to_tensors(tensor_type="""jax"""      ,			prepend_batch_axis=__UpperCAmelCase    )
						self.assertEqual(tensor_batch["""inputs"""].shape      ,			(1, 3)    )
						self.assertEqual(tensor_batch["""labels"""].shape      ,			(1,)    )

			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      List[Any]:
						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": np.array([0, 1, 2]    )}, {"""input_ids""": np.array([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			np.ndarray    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""np"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			np.ndarray    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_torch
			def    SCREAMING_SNAKE_CASE			(   self			:  Dict    )				->      Optional[int]:
						import torch

						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": torch.tensor([0, 1, 2]    )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			torch.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""pt"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			torch.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_tf
			def    SCREAMING_SNAKE_CASE			(   self			:  List[Any]    )				->      Dict:
						import tensorflow as tf

						SCREAMING_SNAKE_CASE__									= [{"""input_ids""": tf.constant([0, 1, 2]    )}, {"""input_ids""": tf.constant([0, 1, 2, 3]    )}]
						SCREAMING_SNAKE_CASE__									= BertTokenizer.from_pretrained("""bert-base-cased"""    )

						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			tf.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )
						SCREAMING_SNAKE_CASE__									= tokenizer.pad(__UpperCAmelCase      ,			padding=__UpperCAmelCase      ,			return_tensors="""tf"""    )
						self.assertTrue(isinstance(batch["""input_ids"""]      ,			tf.Tensor    )    )
						self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]    )

			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  Optional[Any]    )				->      Optional[int]:
						SCREAMING_SNAKE_CASE__									= Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
						PreTrainedTokenizerFast(tokenizer_object=__UpperCAmelCase    )





			@require_tokenizers
			def    SCREAMING_SNAKE_CASE			(   self			:  str    )				->      int:
						SCREAMING_SNAKE_CASE__									= Tokenizer(WordPiece(unk_token="""[UNK]"""    )    )
						with tempfile.TemporaryDirectory() as tmpdirname:
									bert_tokenizer.save(os.path.join(__UpperCAmelCase      ,			"""tokenizer.json"""    )    )
									PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCAmelCase      ,			"""tokenizer.json"""    )    )


