



'''simple docstring'''






import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
						from .tokenization_deberta_va import DebertaVaTokenizer
else:
						UpperCamelCase__:       List[Any]							     =      None

UpperCamelCase__:       Optional[int]							     =      logging.get_logger(__name__)

UpperCamelCase__:       str							     =      {"vocab_file": "spm.model", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__:       str							     =      {
    "vocab_file": {
        "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model"
        ),
        "microsoft/deberta-v2-xxlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model"
        ),
    }
}

UpperCamelCase__:       List[Any]							     =      {
    "microsoft/deberta-v2-xlarge": 512,
    "microsoft/deberta-v2-xxlarge": 512,
    "microsoft/deberta-v2-xlarge-mnli": 512,
    "microsoft/deberta-v2-xxlarge-mnli": 512,
}

UpperCamelCase__:       int							     =      {
    "microsoft/deberta-v2-xlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xlarge-mnli": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge-mnli": {"do_lower_case": False},
}

class     SCREAMING_SNAKE_CASE(							A__      ):







	"""simple docstring"""






	lowerCamelCase__      			=	VOCAB_FILES_NAMES
	lowerCamelCase__      			=	PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__      			=	PRETRAINED_INIT_CONFIGURATION
	lowerCamelCase__      			=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__      			=	DebertaVaTokenizer







	def __init__(						self      :					Optional[int]		,      __snake_case      :					str=None		,      __snake_case      :					Optional[Any]=None		,      __snake_case      :					List[Any]=False		,      __snake_case      :					Tuple=False		,      __snake_case      :					Optional[Any]="[CLS]"		,      __snake_case      :					Any="[SEP]"		,      __snake_case      :					str="[UNK]"		,      __snake_case      :					Tuple="[SEP]"		,      __snake_case      :					int="[PAD]"		,      __snake_case      :					Union[str, Any]="[CLS]"		,      __snake_case      :					List[str]="[MASK]"		,      **__snake_case      :					Dict		,      )      ->		None:
								super().__init__(
								    __snake_case		,      tokenizer_file=__snake_case		,      do_lower_case=__snake_case		,      bos_token=__snake_case		,      eos_token=__snake_case		,      unk_token=__snake_case		,      sep_token=__snake_case		,      pad_token=__snake_case		,      cls_token=__snake_case		,      mask_token=__snake_case		,      split_by_punct=__snake_case		,      **__snake_case		,      )

								UpperCAmelCase			:	Optional[Any]										=		do_lower_case
								UpperCAmelCase			:	Optional[Any]										=		split_by_punct
								UpperCAmelCase			:	Any										=		vocab_file
								UpperCAmelCase			:	Any										=		False if not self.vocab_file else True







	def 					A     (						self      :					Tuple		,      __snake_case      :					Optional[int]		,      __snake_case      :					List[Any]=None		)      ->		Dict:

								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCAmelCase			:	Dict										=		[self.cls_token_id]
								UpperCAmelCase			:	Tuple										=		[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep







	def 					A     (						self      :					Tuple		,      __snake_case      :					Optional[Any]		,      __snake_case      :					Dict=None		,      __snake_case      :					Optional[int]=False		)      ->		Optional[int]:

								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__snake_case		,      token_ids_a=__snake_case		,      already_has_special_tokens=__snake_case		)

								if token_ids_a is not None:
															return [1] + ([0] * len(__snake_case		)) + [1] + ([0] * len(__snake_case		)) + [1]
								return [1] + ([0] * len(__snake_case		)) + [1]







	def 					A     (						self      :					Union[str, Any]		,      __snake_case      :					List[str]		,      __snake_case      :					Optional[Any]=None		)      ->		Union[str, Any]:
								UpperCAmelCase			:	List[Any]										=		[self.sep_token_id]
								UpperCAmelCase			:	Tuple										=		[self.cls_token_id]
								if token_ids_a is None:
															return len(cls + token_ids_a + sep		) * [0]
								return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def 					A     (						self      :					Optional[int]		,      __snake_case      :					str		,      __snake_case      :					Optional[str] = None		)      ->		Tuple[str]:
								if not self.can_save_slow_tokenizer:
															raise ValueError(
															    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
															    '''tokenizer.'''		)

								if not os.path.isdir(__snake_case		):
															logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
															return
								UpperCAmelCase			:	List[str]										=		os.path.join(
								    __snake_case		,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

								if os.path.abspath(self.vocab_file		) != os.path.abspath(__snake_case		):
															copyfile(self.vocab_file		,      __snake_case		)

								return (out_vocab_file,)










'''simple docstring'''






import re


def 			snake_case_					( _lowerCAmelCase  :						List[str]  )				->       Dict:
							def docstring_decorator(_lowerCAmelCase  :						List[str]  ):
														UpperCAmelCase			:	str										=		fn.__doc__
														UpperCAmelCase			:	List[str]										=		func_doc.split('''\n'''  )
														UpperCAmelCase			:	Optional[Any]										=		0
														while i < len(_lowerCAmelCase  ) and re.search(R'''^\s*Examples?:\s*$'''       ,				lines[i]  ) is None:
																					i += 1
														if i < len(_lowerCAmelCase  ):
																					UpperCAmelCase			:	Union[str, Any]										=		example_docstring
																					UpperCAmelCase			:	Tuple										=		'''\n'''.join(_lowerCAmelCase  )
														else:
																					raise ValueError(
																					    f"""The function {fn} should have an empty 'Examples:' in its docstring as placeholder, """
																					    f"""current docstring is:\n{func_doc}"""  )
														UpperCAmelCase			:	Any										=		func_doc
														return fn

							return docstring_decorator





