




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__a  :		Union[str, Any]		     =   logging.get_logger(__name__)


__a  :		List[Any]		     =   {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
__a  :		List[str]		     =   {
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
}

__a  :		Optional[int]		     =   {
    '''RUCAIBox/mvp''': 1_0_2_4,
}
@lru_cache()
def  snake_case_					(		)      ->			List[str]:
							lowercase__  :	Any        =							(
							    list(range(ord("!"    )    ,ord("~"    ) + 1    )    ) + list(range(ord("¡"    )    ,ord("¬"    ) + 1    )    ) + list(range(ord("®"    )    ,ord("ÿ"    ) + 1    )    )
							)
							lowercase__  :	Tuple        =							bs[:]
							lowercase__  :	Tuple        =							0
							for b in range(2**8    ):
														if b not in bs:
																					bs.append(SCREAMING_SNAKE_CASE_    )
																					cs.append(2**8 + n    )
																					n += 1
							lowercase__  :	Optional[Any]        =							[chr(SCREAMING_SNAKE_CASE_    ) for n in cs]
							return dict(zip(SCREAMING_SNAKE_CASE_    ,SCREAMING_SNAKE_CASE_    )    )
def  snake_case_					(		SCREAMING_SNAKE_CASE_    )      ->			Any:
							lowercase__  :	Dict        =							set()
							lowercase__  :	Optional[Any]        =							word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)    )
														lowercase__  :	int        =							char
							return pairs







class 				UpperCAmelCase(					snake_case_					):



	"""simple docstring"""






	a     : Any	  =					VOCAB_FILES_NAMES
	a     : Any	  =					PRETRAINED_VOCAB_FILES_MAP
	a     : Optional[int]	  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a     : Any	  =					["""input_ids""", """attention_mask"""]

	def __init__(					self	,				lowerCamelCase	,				lowerCamelCase	,				lowerCamelCase="replace"	,				lowerCamelCase="<s>"	,				lowerCamelCase="</s>"	,				lowerCamelCase="</s>"	,				lowerCamelCase="<s>"	,				lowerCamelCase="<unk>"	,				lowerCamelCase="<pad>"	,				lowerCamelCase="<mask>"	,				lowerCamelCase=False	,				**lowerCamelCase	,				) ->      List[str]:





								"""simple docstring"""


								lowercase__  :	Tuple        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else bos_token
								lowercase__  :	Optional[int]        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else eos_token
								lowercase__  :	Tuple        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else sep_token
								lowercase__  :	str        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else cls_token
								lowercase__  :	List[str]        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else unk_token
								lowercase__  :	Any        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								lowercase__  :	str        =							AddedToken(lowerCamelCase	,				lstrip=lowerCamelCase	,				rstrip=lowerCamelCase    ) if isinstance(lowerCamelCase	,				lowerCamelCase    ) else mask_token

								super().__init__(
								    errors=lowerCamelCase	,				bos_token=lowerCamelCase	,				eos_token=lowerCamelCase	,				unk_token=lowerCamelCase	,				sep_token=lowerCamelCase	,				cls_token=lowerCamelCase	,				pad_token=lowerCamelCase	,				mask_token=lowerCamelCase	,				add_prefix_space=lowerCamelCase	,				**lowerCamelCase	,				)
								with open(lowerCamelCase	,				encoding="utf-8"    ) as vocab_handle:
															lowercase__  :	Any        =							json.load(lowerCamelCase    )
								lowercase__  :	List[str]        =							{v: k for k, v in self.encoder.items()}
								lowercase__  :	Optional[int]        =							errors  # how to handle errors in decoding
								lowercase__  :	int        =							bytes_to_unicode()
								lowercase__  :	Optional[Any]        =							{v: k for k, v in self.byte_encoder.items()}
								with open(lowerCamelCase	,				encoding="utf-8"    ) as merges_handle:
															lowercase__  :	Any        =							merges_handle.read().split("\n"    )[1:-1]
								lowercase__  :	Any        =							[tuple(merge.split()    ) for merge in bpe_merges]
								lowercase__  :	Union[str, Any]        =							dict(zip(lowerCamelCase	,				range(len(lowerCamelCase    )    )    )    )
								lowercase__  :	str        =							{}
								lowercase__  :	Union[str, Any]        =							add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								lowercase__  :	str        =							re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"    )

	@property
	def    __a     (					self    ) ->      Tuple:





								"""simple docstring"""


								return len(self.encoder    )

	def    __a     (					self    ) ->      int:





								"""simple docstring"""


								return dict(self.encoder	,				**self.added_tokens_encoder    )

	def    __a     (					self	,				lowerCamelCase    ) ->      Optional[Any]:





								"""simple docstring"""


								if token in self.cache:
															return self.cache[token]
								lowercase__  :	Union[str, Any]        =							tuple(lowerCamelCase    )
								lowercase__  :	str        =							get_pairs(lowerCamelCase    )

								if not pairs:
															return token

								while True:
															lowercase__  :	Optional[int]        =							min(lowerCamelCase	,				key=lambda lowerCamelCase    : self.bpe_ranks.get(lowerCamelCase	,				float("inf"    )    )    )
															if bigram not in self.bpe_ranks:
																						break
															lowercase__ ,       lowercase__  :	Tuple        =							bigram
															lowercase__  :	Optional[Any]        =							[]
															lowercase__  :	Optional[int]        =							0
															while i < len(lowerCamelCase    ):
																						try:
																													lowercase__  :	Union[str, Any]        =							word.index(lowerCamelCase	,				lowerCamelCase    )
																						except ValueError:
																													new_word.extend(word[i:]    )
																													break
																						else:
																													new_word.extend(word[i:j]    )
																													lowercase__  :	Any        =							j

																						if word[i] == first and i < len(lowerCamelCase    ) - 1 and word[i + 1] == second:
																													new_word.append(first + second    )
																													i += 2
																						else:
																													new_word.append(word[i]    )
																													i += 1
															lowercase__  :	Optional[Any]        =							tuple(lowerCamelCase    )
															lowercase__  :	List[str]        =							new_word
															if len(lowerCamelCase    ) == 1:
																						break
															else:
																						lowercase__  :	Optional[int]        =							get_pairs(lowerCamelCase    )
								lowercase__  :	Any        =							" ".join(lowerCamelCase    )
								lowercase__  :	Tuple        =							word
								return word

	def    __a     (					self	,				lowerCamelCase    ) ->      Optional[int]:





								"""simple docstring"""


								lowercase__  :	List[str]        =							[]
								for token in re.findall(self.pat	,				lowerCamelCase    ):
															lowercase__  :	Any        =							"".join(
															    self.byte_encoder[b] for b in token.encode("utf-8"    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase    ).split(" "    )    )
								return bpe_tokens

	def    __a     (					self	,				lowerCamelCase    ) ->      Optional[int]:





								"""simple docstring"""


								return self.encoder.get(lowerCamelCase	,				self.encoder.get(self.unk_token    )    )

	def    __a     (					self	,				lowerCamelCase    ) ->      int:





								"""simple docstring"""


								return self.decoder.get(lowerCamelCase    )

	def    __a     (					self	,				lowerCamelCase    ) ->      Tuple:





								"""simple docstring"""


								lowercase__  :	Tuple        =							"".join(lowerCamelCase    )
								lowercase__  :	int        =							bytearray([self.byte_decoder[c] for c in text]    ).decode("utf-8"	,				errors=self.errors    )
								return text

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      Tuple[str]:





								"""simple docstring"""


								if not os.path.isdir(lowerCamelCase    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
															return
								lowercase__  :	int        =							os.path.join(
								    lowerCamelCase	,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
								lowercase__  :	Optional[int]        =							os.path.join(
								    lowerCamelCase	,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]    )

								with open(lowerCamelCase	,				"w"	,				encoding="utf-8"    ) as f:
															f.write(json.dumps(self.encoder	,				indent=2	,				sort_keys=lowerCamelCase	,				ensure_ascii=lowerCamelCase    ) + "\n"    )

								lowercase__  :	Dict        =							0
								with open(lowerCamelCase	,				"w"	,				encoding="utf-8"    ) as writer:
															writer.write("#version: 0.2\n"    )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,				key=lambda lowerCamelCase    : kv[1]    ):
																						if index != token_index:
																													logger.warning(
																													    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																													    " Please check that the tokenizer is not corrupted!"    )
																													lowercase__  :	Optional[int]        =							token_index
																						writer.write(" ".join(lowerCamelCase    ) + "\n"    )
																						index += 1

								return vocab_file, merge_file

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      List[int]:





								"""simple docstring"""


								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowercase__  :	Optional[int]        =							[self.cls_token_id]
								lowercase__  :	Any        =							[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None	,				lowerCamelCase = False    ) ->      List[int]:





								"""simple docstring"""


								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCamelCase	,				token_ids_a=lowerCamelCase	,				already_has_special_tokens=lowerCamelCase    )

								if token_ids_a is None:
															return [1] + ([0] * len(lowerCamelCase    )) + [1]
								return [1] + ([0] * len(lowerCamelCase    )) + [1, 1] + ([0] * len(lowerCamelCase    )) + [1]

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      List[int]:





								"""simple docstring"""


								lowercase__  :	List[str]        =							[self.sep_token_id]
								lowercase__  :	str        =							[self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]







	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=False	,				**lowerCamelCase    ) ->      Any:





								"""simple docstring"""


								lowercase__  :	Optional[Any]        =							kwargs.pop("add_prefix_space"	,				self.add_prefix_space    )
								if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase    ) > 0 and not text[0].isspace()):
															lowercase__  :	Optional[Any]        =							" " + text
								return (text, kwargs)





import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
						from .tokenization_deberta_va import DebertaVaTokenizer
else:
						__a  :		Optional[Any]		     =   None

__a  :		List[str]		     =   logging.get_logger(__name__)

__a  :		Dict		     =   {'''vocab_file''': '''spm.model''', '''tokenizer_file''': '''tokenizer.json'''}

__a  :		int		     =   {
    '''vocab_file''': {
        '''microsoft/deberta-v2-xlarge''': '''https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model''',
        '''microsoft/deberta-v2-xxlarge''': '''https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model''',
        '''microsoft/deberta-v2-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'''
        ),
        '''microsoft/deberta-v2-xxlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'''
        ),
    }
}

__a  :		Any		     =   {
    '''microsoft/deberta-v2-xlarge''': 5_1_2,
    '''microsoft/deberta-v2-xxlarge''': 5_1_2,
    '''microsoft/deberta-v2-xlarge-mnli''': 5_1_2,
    '''microsoft/deberta-v2-xxlarge-mnli''': 5_1_2,
}

__a  :		Union[str, Any]		     =   {
    '''microsoft/deberta-v2-xlarge''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xxlarge''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xlarge-mnli''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xxlarge-mnli''': {'''do_lower_case''': False},
}







class 				UpperCAmelCase(					snake_case_					):



	"""simple docstring"""






	a     : List[Any]	  =					VOCAB_FILES_NAMES
	a     : Optional[int]	  =					PRETRAINED_VOCAB_FILES_MAP
	a     : int	  =					PRETRAINED_INIT_CONFIGURATION
	a     : List[str]	  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a     : int	  =					DebertaVaTokenizer

	def __init__(					self	,				lowerCamelCase=None	,				lowerCamelCase=None	,				lowerCamelCase=False	,				lowerCamelCase=False	,				lowerCamelCase="[CLS]"	,				lowerCamelCase="[SEP]"	,				lowerCamelCase="[UNK]"	,				lowerCamelCase="[SEP]"	,				lowerCamelCase="[PAD]"	,				lowerCamelCase="[CLS]"	,				lowerCamelCase="[MASK]"	,				**lowerCamelCase	,				) ->      None:





								"""simple docstring"""


								super().__init__(
								    lowerCamelCase	,				tokenizer_file=lowerCamelCase	,				do_lower_case=lowerCamelCase	,				bos_token=lowerCamelCase	,				eos_token=lowerCamelCase	,				unk_token=lowerCamelCase	,				sep_token=lowerCamelCase	,				pad_token=lowerCamelCase	,				cls_token=lowerCamelCase	,				mask_token=lowerCamelCase	,				split_by_punct=lowerCamelCase	,				**lowerCamelCase	,				)

								lowercase__  :	Optional[int]        =							do_lower_case
								lowercase__  :	Optional[Any]        =							split_by_punct
								lowercase__  :	Tuple        =							vocab_file
								lowercase__  :	Optional[int]        =							False if not self.vocab_file else True

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None    ) ->      List[Any]:





								"""simple docstring"""


								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowercase__  :	Union[str, Any]        =							[self.cls_token_id]
								lowercase__  :	Union[str, Any]        =							[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None	,				lowerCamelCase=False    ) ->      Union[str, Any]:





								"""simple docstring"""


								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCamelCase	,				token_ids_a=lowerCamelCase	,				already_has_special_tokens=lowerCamelCase    )

								if token_ids_a is not None:
															return [1] + ([0] * len(lowerCamelCase    )) + [1] + ([0] * len(lowerCamelCase    )) + [1]
								return [1] + ([0] * len(lowerCamelCase    )) + [1]

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None    ) ->      Optional[int]:





								"""simple docstring"""


								lowercase__  :	Any        =							[self.sep_token_id]
								lowercase__  :	List[str]        =							[self.cls_token_id]
								if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      Tuple[str]:





								"""simple docstring"""


								if not self.can_save_slow_tokenizer:
															raise ValueError(
															    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
															    "tokenizer."    )

								if not os.path.isdir(lowerCamelCase    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
															return
								lowercase__  :	int        =							os.path.join(
								    lowerCamelCase	,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

								if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase    ):
															copyfile(self.vocab_file	,				lowerCamelCase    )

								return (out_vocab_file,)