




import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

_a				     =  logging.get_logger(__name__)

_a				     =  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

_a				     =  {
    '''vocab_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'''
        ),
    },
}

_a				     =  {
    '''microsoft/deberta-base''': 512,
    '''microsoft/deberta-large''': 512,
    '''microsoft/deberta-xlarge''': 512,
    '''microsoft/deberta-base-mnli''': 512,
    '''microsoft/deberta-large-mnli''': 512,
    '''microsoft/deberta-xlarge-mnli''': 512,
}

_a				     =  {
    '''microsoft/deberta-base''': {'''do_lower_case''': False},
    '''microsoft/deberta-large''': {'''do_lower_case''': False},
}


def __A   (				)->  Tuple:





						"""simple docstring"""



						_UpperCAmelCase     =      (
						    list(range(ord('!'     )			,		ord('~'     ) + 1     )     ) + list(range(ord('¡'     )			,		ord('¬'     ) + 1     )     ) + list(range(ord('®'     )			,		ord('ÿ'     ) + 1     )     )
						)
						_UpperCAmelCase     =      bs[:]
						_UpperCAmelCase     =      0
						for b in range(2**8     ):
												if b not in bs:
																		bs.append(__lowerCAmelCase     )
																		cs.append(2**8 + n     )
																		n += 1
						_UpperCAmelCase     =      [chr(__lowerCAmelCase     ) for n in cs]
						return dict(zip(__lowerCAmelCase			,		__lowerCAmelCase     )     )


def __A   (				__lowerCAmelCase     )->  Optional[int]:





						"""simple docstring"""



						_UpperCAmelCase     =      set()
						_UpperCAmelCase     =      word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)     )
												_UpperCAmelCase     =      char
						return pairs





class __lowerCamelCase			(				SCREAMING_SNAKE_CASE_):





	"""simple docstring"""






	UpperCamelCase__								=				VOCAB_FILES_NAMES
	UpperCamelCase__								=				PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase__								=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase__								=				["input_ids", "attention_mask", "token_type_ids"]

	def __init__(							self	,   UpperCAmelCase	,   UpperCAmelCase	,   UpperCAmelCase="replace"	,   UpperCAmelCase="[CLS]"	,   UpperCAmelCase="[SEP]"	,   UpperCAmelCase="[SEP]"	,   UpperCAmelCase="[CLS]"	,   UpperCAmelCase="[UNK]"	,   UpperCAmelCase="[PAD]"	,   UpperCAmelCase="[MASK]"	,   UpperCAmelCase=False	,   UpperCAmelCase=False	,   **UpperCAmelCase	,   ):
							"""simple docstring"""







							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else bos_token
							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else eos_token
							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else sep_token
							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else cls_token
							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else unk_token
							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							_UpperCAmelCase     =      AddedToken(lowercase_	,   lstrip=lowercase_	,   rstrip=lowercase_   ) if isinstance(lowercase_	,   lowercase_   ) else mask_token

							super().__init__(
							    errors=lowercase_	,   bos_token=lowercase_	,   eos_token=lowercase_	,   unk_token=lowercase_	,   sep_token=lowercase_	,   cls_token=lowercase_	,   pad_token=lowercase_	,   mask_token=lowercase_	,   add_prefix_space=lowercase_	,   add_bos_token=lowercase_	,   **lowercase_	,   )
							_UpperCAmelCase     =      add_bos_token

							with open(lowercase_	,   encoding='utf-8'   ) as vocab_handle:
													_UpperCAmelCase     =      json.load(lowercase_   )
							_UpperCAmelCase     =      {v: k for k, v in self.encoder.items()}
							_UpperCAmelCase     =      errors  # how to handle errors in decoding
							_UpperCAmelCase     =      bytes_to_unicode()
							_UpperCAmelCase     =      {v: k for k, v in self.byte_encoder.items()}
							with open(lowercase_	,   encoding='utf-8'   ) as merges_handle:
													_UpperCAmelCase     =      merges_handle.read().split('\n'   )[1:-1]
							_UpperCAmelCase     =      [tuple(merge.split()   ) for merge in bpe_merges]
							_UpperCAmelCase     =      dict(zip(lowercase_	,   range(len(lowercase_   )   )   )   )
							_UpperCAmelCase     =      {}
							_UpperCAmelCase     =      add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							_UpperCAmelCase     =      re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'   )

	@property
	# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return len(self.encoder   )

	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return dict(self.encoder	,   **self.added_tokens_encoder   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							if token in self.cache:
													return self.cache[token]
							_UpperCAmelCase     =      tuple(lowercase_   )
							_UpperCAmelCase     =      get_pairs(lowercase_   )

							if not pairs:
													return token

							while True:
													_UpperCAmelCase     =      min(lowercase_	,   key=lambda UpperCAmelCase   : self.bpe_ranks.get(lowercase_	,   float('inf'   )   )   )
													if bigram not in self.bpe_ranks:
																			break
													_UpperCAmelCase ,			_UpperCAmelCase     =      bigram
													_UpperCAmelCase     =      []
													_UpperCAmelCase     =      0
													while i < len(lowercase_   ):
																			try:
																									_UpperCAmelCase     =      word.index(lowercase_	,   lowercase_   )
																			except ValueError:
																									new_word.extend(word[i:]   )
																									break
																			else:
																									new_word.extend(word[i:j]   )
																									_UpperCAmelCase     =      j

																			if word[i] == first and i < len(lowercase_   ) - 1 and word[i + 1] == second:
																									new_word.append(first + second   )
																									i += 2
																			else:
																									new_word.append(word[i]   )
																									i += 1
													_UpperCAmelCase     =      tuple(lowercase_   )
													_UpperCAmelCase     =      new_word
													if len(lowercase_   ) == 1:
																			break
													else:
																			_UpperCAmelCase     =      get_pairs(lowercase_   )
							_UpperCAmelCase     =      ' '.join(lowercase_   )
							_UpperCAmelCase     =      word
							return word

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None   ):
							"""simple docstring"""







							if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							_UpperCAmelCase     =      [self.cls_token_id]
							_UpperCAmelCase     =      [self.sep_token_id]
							return cls + token_ids_a + sep + token_ids_a + sep

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None	,   UpperCAmelCase = False   ):
							"""simple docstring"""







							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowercase_	,   token_ids_a=lowercase_	,   already_has_special_tokens=lowercase_   )

							if token_ids_a is None:
													return [1] + ([0] * len(lowercase_   )) + [1]
							return [1] + ([0] * len(lowercase_   )) + [1] + ([0] * len(lowercase_   )) + [1]

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None   ):
							"""simple docstring"""







							_UpperCAmelCase     =      [self.sep_token_id]
							_UpperCAmelCase     =      [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      []
							for token in re.findall(self.pat	,   lowercase_   ):
													_UpperCAmelCase     =      ''.join(
													    self.byte_encoder[b] for b in token.encode('utf-8'   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
													bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_   ).split(' '   )   )
							return bpe_tokens

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							return self.encoder.get(lowercase_	,   self.encoder.get(self.unk_token   )   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							return self.decoder.get(lowercase_   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      ''.join(lowercase_   )
							_UpperCAmelCase     =      bytearray([self.byte_decoder[c] for c in text]   ).decode('utf-8'	,   errors=self.errors   )
							return text

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase = None   ):
							"""simple docstring"""







							if not os.path.isdir(lowercase_   ):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
													return
							_UpperCAmelCase     =      os.path.join(
							    lowercase_	,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
							_UpperCAmelCase     =      os.path.join(
							    lowercase_	,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

							with open(lowercase_	,   'w'	,   encoding='utf-8'   ) as f:
													f.write(json.dumps(self.encoder	,   indent=2	,   sort_keys=lowercase_	,   ensure_ascii=lowercase_   ) + '\n'   )

							_UpperCAmelCase     =      0
							with open(lowercase_	,   'w'	,   encoding='utf-8'   ) as writer:
													writer.write('#version: 0.2\n'   )
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,   key=lambda UpperCAmelCase   : kv[1]   ):
																			if index != token_index:
																									logger.warning(
																									    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																									    ' Please check that the tokenizer is not corrupted!'   )
																									_UpperCAmelCase     =      token_index
																			writer.write(' '.join(lowercase_   ) + '\n'   )
																			index += 1

							return vocab_file, merge_file

	def 	UpperCamelCase   (							self	,   UpperCAmelCase	,   UpperCAmelCase=False	,   **UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      kwargs.pop('add_prefix_space'	,   self.add_prefix_space   )
							if (is_split_into_words or add_prefix_space) and (len(lowercase_   ) > 0 and not text[0].isspace()):
													_UpperCAmelCase     =      ' ' + text
							return (text, kwargs)

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      []
							for is_user, text in conversation.iter_texts():
													input_ids.extend(self.encode(lowercase_	,   add_special_tokens=lowercase_   ) + [self.eos_token_id]   )
							if len(lowercase_   ) > self.model_max_length:
													_UpperCAmelCase     =      input_ids[-self.model_max_length :]
							return input_ids








from functools import partial
from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload

import torch
import torch.nn as nn
import torch.types


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  torch.Tensor:





						"""simple docstring"""



						if not inplace:
												_UpperCAmelCase     =      ma + ma
						else:
												ma += ma

						return ma


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase     )->  torch.Tensor:





						"""simple docstring"""



						_UpperCAmelCase     =      -1 * len(__lowerCAmelCase     )
						_UpperCAmelCase     =      list(range(len(tensor.shape[:zero_index]     )     )     )
						return tensor.permute(first_inds + [zero_index + i for i in inds]     )


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase     )->  torch.Tensor:





						"""simple docstring"""



						return t.reshape(t.shape[:-no_dims] + (-1,)     )


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase = 1E-4     )->  torch.Tensor:





						"""simple docstring"""



						_UpperCAmelCase     =      mask.expand(*value.shape     )
						return torch.sum(mask * value			,		dim=__lowerCAmelCase     ) / (eps + torch.sum(__lowerCAmelCase			,		dim=__lowerCAmelCase     ))


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase = 2.31_25			,		__lowerCAmelCase = 21.68_75			,		__lowerCAmelCase = 64     )->  torch.Tensor:





						"""simple docstring"""



						_UpperCAmelCase     =      torch.linspace(__lowerCAmelCase			,		__lowerCAmelCase			,		no_bins - 1			,		device=pts.device     )
						_UpperCAmelCase     =      torch.sqrt(torch.sum((pts.unsqueeze(-2     ) - pts.unsqueeze(-3     )) ** 2			,		dim=-1     )     )
						return torch.bucketize(__lowerCAmelCase			,		__lowerCAmelCase     )


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase     )->  dict:





						"""simple docstring"""



						_UpperCAmelCase     =      dicts[0]
						_UpperCAmelCase     =      {}
						for k, v in first.items():
												_UpperCAmelCase     =      [d[k] for d in dicts]
												if isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
																		_UpperCAmelCase     =      dict_multimap(__lowerCAmelCase			,		__lowerCAmelCase     )
												else:
																		_UpperCAmelCase     =      fn(__lowerCAmelCase     )

						return new_dict


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase     )->  torch.Tensor:





						"""simple docstring"""



						_UpperCAmelCase     =      v_bins.view(((1,) * len(x.shape     )) + (len(__lowerCAmelCase     ),)     )
						_UpperCAmelCase     =      x[..., None] - reshaped_bins
						_UpperCAmelCase     =      torch.argmin(torch.abs(__lowerCAmelCase     )			,		dim=-1     )
						return nn.functional.one_hot(__lowerCAmelCase			,		num_classes=len(__lowerCAmelCase     )     ).float()


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase = 0			,		__lowerCAmelCase = 0     )->  torch.Tensor:





						"""simple docstring"""



						_UpperCAmelCase     =      []
						for i, s in enumerate(data.shape[:no_batch_dims]     ):
												_UpperCAmelCase     =      torch.arange(__lowerCAmelCase     )
												_UpperCAmelCase     =      r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape     ) - i - 1)))     )
												ranges.append(__lowerCAmelCase     )

						_UpperCAmelCase     =      [slice(__lowerCAmelCase     ) for _ in range(len(data.shape     ) - no_batch_dims     )]
						_UpperCAmelCase     =      inds
						ranges.extend(__lowerCAmelCase     )
						# Matt note: Editing this to get around the behaviour of using a list as an array index changing
						# in recent Numpy versions
						return data[tuple(__lowerCAmelCase     )]


_a				     =  TypeVar('''T''')


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  Dict[Any, Union[dict, list, tuple, Any]]:





						"""simple docstring"""



						_UpperCAmelCase     =      {}
						for k, v in dic.items():
												if isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
																		_UpperCAmelCase     =      dict_map(__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )
												else:
																		_UpperCAmelCase     =      tree_map(__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )

						return new_dict


@overload
def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  Any:





						"""simple docstring"""



						...


@overload
def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  dict:





						"""simple docstring"""



						...


@overload
def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  list:





						"""simple docstring"""



						...


@overload
def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  tuple:





						"""simple docstring"""



						...


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )->  List[Any]:





						"""simple docstring"""



						if isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
												return dict_map(__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )
						elif isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
												return [tree_map(__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     ) for x in tree]
						elif isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
												return tuple(tree_map(__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     ) for x in tree     )
						elif isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
												return fn(__lowerCAmelCase     )
						else:
												print(type(__lowerCAmelCase     )     )
												raise ValueError('Not supported'     )

_a				     =  partial(tree_map, leaf_type=torch.Tensor)
