


'''simple docstring'''


import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation

UpperCAmelCase_		:       Tuple							       =		logging.get_logger(__name__)

UpperCAmelCase_		:       Dict							       =		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

UpperCAmelCase_		:       List[str]							       =		{
    '''vocab_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'''
        ),
    },
}

UpperCAmelCase_		:       Union[str, Any]							       =		{
    '''microsoft/deberta-base''': 512,
    '''microsoft/deberta-large''': 512,
    '''microsoft/deberta-xlarge''': 512,
    '''microsoft/deberta-base-mnli''': 512,
    '''microsoft/deberta-large-mnli''': 512,
    '''microsoft/deberta-xlarge-mnli''': 512,
}

UpperCAmelCase_		:       int							       =		{
    '''microsoft/deberta-base''': {'''do_lower_case''': False},
    '''microsoft/deberta-large''': {'''do_lower_case''': False},
}







def       snake_case_       (  ):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	Any							=				(
				    list(range(ord("""!"""					)							,				ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					)							,				ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					)							,				ord("""ÿ"""					) + 1					)					)
				)
				_SCREAMING_SNAKE_CASE			:	Tuple							=				bs[:]
				_SCREAMING_SNAKE_CASE			:	Dict							=				0
				for b in range(2**8					):
								if b not in bs:
												bs.append(A__					)
												cs.append(2**8 + n					)
												n += 1
				_SCREAMING_SNAKE_CASE			:	str							=				[chr(A__					) for n in cs]
				return dict(zip(A__							,				A__					)					)







def       snake_case_       (  SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	str							=				set()
				_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)					)
								_SCREAMING_SNAKE_CASE			:	List[str]							=				char
				return pairs







class    lowercase__  (						_lowerCamelCase ):





							'''simple docstring'''


							A_							:				str				     =  VOCAB_FILES_NAMES
							A_							:				Optional[int]				     =  PRETRAINED_VOCAB_FILES_MAP
							A_							:				Optional[int]				     =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_							:				Tuple				     =  ["""input_ids""", """attention_mask""", """token_type_ids"""]






							def __init__(   self   , __snake_case   , __snake_case   , __snake_case="replace"   , __snake_case="[CLS]"   , __snake_case="[SEP]"   , __snake_case="[SEP]"   , __snake_case="[CLS]"   , __snake_case="[UNK]"   , __snake_case="[PAD]"   , __snake_case="[MASK]"   , __snake_case=False   , __snake_case=False   , **__snake_case   , ):
											_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else bos_token
											_SCREAMING_SNAKE_CASE			:	Dict							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else eos_token
											_SCREAMING_SNAKE_CASE			:	Optional[int]							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else sep_token
											_SCREAMING_SNAKE_CASE			:	Any							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else cls_token
											_SCREAMING_SNAKE_CASE			:	str							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else unk_token
											_SCREAMING_SNAKE_CASE			:	Dict							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				AddedToken(__UpperCamelCase   , lstrip=__UpperCamelCase   , rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase   , __UpperCamelCase  ) else mask_token

											super().__init__(
											    errors=__UpperCamelCase   , bos_token=__UpperCamelCase   , eos_token=__UpperCamelCase   , unk_token=__UpperCamelCase   , sep_token=__UpperCamelCase   , cls_token=__UpperCamelCase   , pad_token=__UpperCamelCase   , mask_token=__UpperCamelCase   , add_prefix_space=__UpperCamelCase   , add_bos_token=__UpperCamelCase   , **__UpperCamelCase   , )
											_SCREAMING_SNAKE_CASE			:	int							=				add_bos_token

											with open(__UpperCamelCase   , encoding="""utf-8"""  ) as vocab_handle:
															_SCREAMING_SNAKE_CASE			:	int							=				json.load(__UpperCamelCase  )
											_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				{v: k for k, v in self.encoder.items()}
											_SCREAMING_SNAKE_CASE			:	Optional[int]							=				errors  # how to handle errors in decoding
											_SCREAMING_SNAKE_CASE			:	str							=				bytes_to_unicode()
											_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				{v: k for k, v in self.byte_encoder.items()}
											with open(__UpperCamelCase   , encoding="""utf-8"""  ) as merges_handle:
															_SCREAMING_SNAKE_CASE			:	Optional[int]							=				merges_handle.read().split("""\n"""  )[1:-1]
											_SCREAMING_SNAKE_CASE			:	Tuple							=				[tuple(merge.split()  ) for merge in bpe_merges]
											_SCREAMING_SNAKE_CASE			:	Optional[int]							=				dict(zip(__UpperCamelCase   , range(len(__UpperCamelCase  )  )  )  )
											_SCREAMING_SNAKE_CASE			:	str							=				{}
											_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											_SCREAMING_SNAKE_CASE			:	List[str]							=				re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )






							@property
							# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
							def UpperCAmelCase_     (   self  ):
											return len(self.encoder  )






							def UpperCAmelCase_     (   self  ):
											return dict(self.encoder   , **self.added_tokens_encoder  )






							def UpperCAmelCase_     (   self   , __snake_case  ):







											if token in self.cache:
															return self.cache[token]
											_SCREAMING_SNAKE_CASE			:	Any							=				tuple(__UpperCamelCase  )
											_SCREAMING_SNAKE_CASE			:	int							=				get_pairs(__UpperCamelCase  )

											if not pairs:
															return token

											while True:
															_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				min(__UpperCamelCase   , key=lambda __snake_case  : self.bpe_ranks.get(__UpperCamelCase   , float("""inf"""  )  )  )
															if bigram not in self.bpe_ranks:
																			break
															_SCREAMING_SNAKE_CASE			:	Any							=				bigram
															_SCREAMING_SNAKE_CASE			:	Any							=				[]
															_SCREAMING_SNAKE_CASE			:	int							=				0
															while i < len(__UpperCamelCase  ):
																			try:
																							_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				word.index(__UpperCamelCase   , __UpperCamelCase  )
																			except ValueError:
																							new_word.extend(word[i:]  )
																							break
																			else:
																							new_word.extend(word[i:j]  )
																							_SCREAMING_SNAKE_CASE			:	str							=				j

																			if word[i] == first and i < len(__UpperCamelCase  ) - 1 and word[i + 1] == second:
																							new_word.append(first + second  )
																							i += 2
																			else:
																							new_word.append(word[i]  )
																							i += 1
															_SCREAMING_SNAKE_CASE			:	Any							=				tuple(__UpperCamelCase  )
															_SCREAMING_SNAKE_CASE			:	Any							=				new_word
															if len(__UpperCamelCase  ) == 1:
																			break
															else:
																			_SCREAMING_SNAKE_CASE			:	Tuple							=				get_pairs(__UpperCamelCase  )
											_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				' '.join(__UpperCamelCase  )
											_SCREAMING_SNAKE_CASE			:	Tuple							=				word
											return word






							def UpperCAmelCase_     (   self   , __snake_case   , __snake_case = None  ):







											if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				[self.cls_token_id]
											_SCREAMING_SNAKE_CASE			:	int							=				[self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep






							def UpperCAmelCase_     (   self   , __snake_case   , __snake_case = None   , __snake_case = False  ):







											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__UpperCamelCase   , token_ids_a=__UpperCamelCase   , already_has_special_tokens=__UpperCamelCase  )

											if token_ids_a is None:
															return [1] + ([0] * len(__UpperCamelCase  )) + [1]
											return [1] + ([0] * len(__UpperCamelCase  )) + [1] + ([0] * len(__UpperCamelCase  )) + [1]






							def UpperCAmelCase_     (   self   , __snake_case   , __snake_case = None  ):
											_SCREAMING_SNAKE_CASE			:	List[str]							=				[self.sep_token_id]
											_SCREAMING_SNAKE_CASE			:	Any							=				[self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep  ) * [0]
											return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






							def UpperCAmelCase_     (   self   , __snake_case  ):
											_SCREAMING_SNAKE_CASE			:	Dict							=				[]
											for token in re.findall(self.pat   , __UpperCamelCase  ):
															_SCREAMING_SNAKE_CASE			:	Dict							=				''.join(
															    self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCamelCase  ).split(""" """  )  )
											return bpe_tokens






							def UpperCAmelCase_     (   self   , __snake_case  ):
											return self.encoder.get(__UpperCamelCase   , self.encoder.get(self.unk_token  )  )






							def UpperCAmelCase_     (   self   , __snake_case  ):
											return self.decoder.get(__UpperCamelCase  )






							def UpperCAmelCase_     (   self   , __snake_case  ):
											_SCREAMING_SNAKE_CASE			:	int							=				''.join(__UpperCamelCase  )
											_SCREAMING_SNAKE_CASE			:	Optional[int]							=				bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8"""   , errors=self.errors  )
											return text






							def UpperCAmelCase_     (   self   , __snake_case   , __snake_case = None  ):







											if not os.path.isdir(__UpperCamelCase  ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
															return
											_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				os.path.join(
											    __UpperCamelCase   , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
											_SCREAMING_SNAKE_CASE			:	List[Any]							=				os.path.join(
											    __UpperCamelCase   , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

											with open(__UpperCamelCase   , """w"""   , encoding="""utf-8"""  ) as f:
															f.write(json.dumps(self.encoder   , indent=2   , sort_keys=__UpperCamelCase   , ensure_ascii=__UpperCamelCase  ) + """\n"""  )

											_SCREAMING_SNAKE_CASE			:	int							=				0
											with open(__UpperCamelCase   , """w"""   , encoding="""utf-8"""  ) as writer:
															writer.write("""#version: 0.2\n"""  )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   , key=lambda __snake_case  : kv[1]  ):
																			if index != token_index:
																							logger.warning(
																							    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																							    """ Please check that the tokenizer is not corrupted!"""  )
																							_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				token_index
																			writer.write(""" """.join(__UpperCamelCase  ) + """\n"""  )
																			index += 1

											return vocab_file, merge_file






							def UpperCAmelCase_     (   self   , __snake_case   , __snake_case=False   , **__snake_case  ):
											_SCREAMING_SNAKE_CASE			:	Dict							=				kwargs.pop("""add_prefix_space"""   , self.add_prefix_space  )
											if (is_split_into_words or add_prefix_space) and (len(__UpperCamelCase  ) > 0 and not text[0].isspace()):
															_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				' ' + text
											return (text, kwargs)






							def UpperCAmelCase_     (   self   , __snake_case  ):
											_SCREAMING_SNAKE_CASE			:	Dict							=				[]
											for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(__UpperCamelCase   , add_special_tokens=__UpperCamelCase  ) + [self.eos_token_id]  )
											if len(__UpperCamelCase  ) > self.model_max_length:
															_SCREAMING_SNAKE_CASE			:	Tuple							=				input_ids[-self.model_max_length :]
											return input_ids








'''simple docstring'''


def       snake_case_       (  SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	Tuple							=				1
				while b > 0:
								if b & 1:
												res *= a

								a *= a
								b >>= 1

				return res







def       snake_case_       (  SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				1
				while b > 0:
								if b & 1:
												_SCREAMING_SNAKE_CASE			:	Tuple							=				((res % c) * (a % c)) % c

								a *= a
								b >>= 1

				return res




