



import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
							from .tokenization_deberta_va import DebertaVaTokenizer
else:
							A__     				=	None

A__     				=	logging.get_logger(__name__)

A__     				=	{"""vocab_file""": """spm.model""", """tokenizer_file""": """tokenizer.json"""}

A__     				=	{
    """vocab_file""": {
        """microsoft/deberta-v2-xlarge""": """https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model""",
        """microsoft/deberta-v2-xxlarge""": """https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model""",
        """microsoft/deberta-v2-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model"""
        ),
        """microsoft/deberta-v2-xxlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model"""
        ),
    }
}

A__     				=	{
    """microsoft/deberta-v2-xlarge""": 5_12,
    """microsoft/deberta-v2-xxlarge""": 5_12,
    """microsoft/deberta-v2-xlarge-mnli""": 5_12,
    """microsoft/deberta-v2-xxlarge-mnli""": 5_12,
}

A__     				=	{
    """microsoft/deberta-v2-xlarge""": {"""do_lower_case""": False},
    """microsoft/deberta-v2-xxlarge""": {"""do_lower_case""": False},
    """microsoft/deberta-v2-xlarge-mnli""": {"""do_lower_case""": False},
    """microsoft/deberta-v2-xxlarge-mnli""": {"""do_lower_case""": False},
}





class  __lowerCAmelCase			(					__lowerCamelCase	):
				__lowerCamelCase			=    VOCAB_FILES_NAMES
				__lowerCamelCase			=    PRETRAINED_VOCAB_FILES_MAP
				__lowerCamelCase			=    PRETRAINED_INIT_CONFIGURATION
				__lowerCamelCase			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__lowerCamelCase			=    DebertaVaTokenizer




				def __init__(							self						,    _snake_case=None						,    _snake_case=None						,    _snake_case=False						,    _snake_case=False						,    _snake_case="[CLS]"						,    _snake_case="[SEP]"						,    _snake_case="[UNK]"						,    _snake_case="[SEP]"						,    _snake_case="[PAD]"						,    _snake_case="[CLS]"						,    _snake_case="[MASK]"						,    **_snake_case						,    ):



									"""simple docstring"""





									super().__init__(
									    _UpperCamelCase						,    tokenizer_file=_UpperCamelCase						,    do_lower_case=_UpperCamelCase						,    bos_token=_UpperCamelCase						,    eos_token=_UpperCamelCase						,    unk_token=_UpperCamelCase						,    sep_token=_UpperCamelCase						,    pad_token=_UpperCamelCase						,    cls_token=_UpperCamelCase						,    mask_token=_UpperCamelCase						,    split_by_punct=_UpperCamelCase						,    **_UpperCamelCase						,    )

									_lowerCAmelCase      =   do_lower_case
									_lowerCAmelCase      =   split_by_punct
									_lowerCAmelCase      =   vocab_file
									_lowerCAmelCase      =   False if not self.vocab_file else True




				def 							snake_case       (							self						,    _snake_case						,    _snake_case=None		):



									"""simple docstring"""





									if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_lowerCAmelCase      =   [self.cls_token_id]
									_lowerCAmelCase      =   [self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep




				def 							snake_case       (							self						,    _snake_case						,    _snake_case=None						,    _snake_case=False		):



									"""simple docstring"""





									if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_UpperCamelCase						,    token_ids_a=_UpperCamelCase						,    already_has_special_tokens=_UpperCamelCase		)

									if token_ids_a is not None:
														return [1] + ([0] * len(_UpperCamelCase		)) + [1] + ([0] * len(_UpperCamelCase		)) + [1]
									return [1] + ([0] * len(_UpperCamelCase		)) + [1]




				def 							snake_case       (							self						,    _snake_case						,    _snake_case=None		):



									"""simple docstring"""





									_lowerCAmelCase      =   [self.sep_token_id]
									_lowerCAmelCase      =   [self.cls_token_id]
									if token_ids_a is None:
														return len(cls + token_ids_a + sep		) * [0]
									return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




				def 							snake_case       (							self						,    _snake_case						,    _snake_case = None		):



									"""simple docstring"""





									if not self.can_save_slow_tokenizer:
														raise ValueError(
														    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
														    """tokenizer."""		)

									if not os.path.isdir(_UpperCamelCase		):
														logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
														return
									_lowerCAmelCase      =   os.path.join(
									    _UpperCamelCase						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

									if os.path.abspath(self.vocab_file		) != os.path.abspath(_UpperCamelCase		):
														copyfile(self.vocab_file						,    _UpperCamelCase		)

									return (out_vocab_file,)








# Lint as: python3

import hashlib
from typing import Union


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					if isinstance(snake_case     ,						snake_case      ):
										# Data already in bytes, returns as it as
										return hash_data
					elif isinstance(snake_case     ,						snake_case      ):
										# We keep the data as it as for it ot be later encoded to UTF-8
										# However replace `\\` with `/` for Windows compatibility
										_lowerCAmelCase      =   hash_data.replace("""\\"""     ,						"""/"""      )
					elif isinstance(snake_case     ,						snake_case      ):
										_lowerCAmelCase      =   str(snake_case      )
					else:
										# If data is not of the required type, raise error
										raise InvalidKeyError(snake_case      )

					return hash_data.encode("""utf-8"""      )





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   """\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
									_lowerCAmelCase      =   F'\nFound Key {hash_data} of type {type(_snake_case		)}'
									_lowerCAmelCase      =   """\nKeys should be either str, int or bytes type"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case						,    _snake_case						,    _snake_case=""		):



									"""simple docstring"""





									_lowerCAmelCase      =   key
									_lowerCAmelCase      =   duplicate_key_indices
									_lowerCAmelCase      =   fix_msg
									_lowerCAmelCase      =   """Found multiple examples generated with the same key"""
									if len(_snake_case		) <= 20:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(_snake_case		)} have the key {key}'
									else:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(duplicate_key_indices[:20]		)}... ({len(_snake_case		) - 20} more) have the key {key}'
									_lowerCAmelCase      =   """\n""" + fix_msg if fix_msg else """"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			:




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   hashlib.mda(_as_bytes(_snake_case		)		)




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   self._split_mda.copy()
									_lowerCAmelCase      =   _as_bytes(_snake_case		)
									mda.update(_snake_case		)
									# Convert to integer with hexadecimal conversion
									return int(mda.hexdigest()						,    16		)



