



'''simple docstring'''





import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

A			:   Optional[Any]			=      logging.get_logger(__name__)

A			:   str			=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

A			:   Optional[Any]			=      {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

A			:   str			=      {
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

A			:   List[str]			=      {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}
def 		_a   (  ):
   snake_case   :							Optional[int]						  =(
       list(range(ord('''!'''							)					,						ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)					,						ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)					,						ord('''ÿ'''							) + 1							)							)
   )
   snake_case   :							Tuple						  =bs[:]
   snake_case   :							Union[str, Any]						  =0
   for b in range(2**8							):
      if b not in bs:
         bs.append(SCREAMING_SNAKE_CASE__							)
         cs.append(2**8 + n							)
         n += 1
   snake_case   :							List[Any]						  =[chr(SCREAMING_SNAKE_CASE__							) for n in cs]
   return dict(zip(SCREAMING_SNAKE_CASE__					,						SCREAMING_SNAKE_CASE__							)							)
def 		_a   (  lowerCamelCase_							):
   snake_case   :							Tuple						  =set()
   snake_case   :							str						  =word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)							)
      snake_case   :							str						  =char
   return pairs





class 							lowerCAmelCase_		(	UpperCamelCase_						):
    __UpperCAmelCase							=	VOCAB_FILES_NAMES
    __UpperCAmelCase							=	PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase							=	['input_ids', 'attention_mask', 'token_type_ids']





    def __init__(    self    :    Any,    _snake_case    :    Union[str, Any],    _snake_case    :    Optional[Any],    _snake_case    :    List[Any]="replace",    _snake_case    :    str="[CLS]",    _snake_case    :    Any="[SEP]",    _snake_case    :    str="[SEP]",    _snake_case    :    List[str]="[CLS]",    _snake_case    :    Dict="[UNK]",    _snake_case    :    str="[PAD]",    _snake_case    :    Union[str, Any]="[MASK]",    _snake_case    :    List[Any]=False,    _snake_case    :    List[str]=False,    **_snake_case    :    Tuple,    ):



       '''simple docstring'''







       snake_case   :							Union[str, Any]						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else bos_token
       snake_case   :							int						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else eos_token
       snake_case   :							List[str]						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else sep_token
       snake_case   :							Optional[Any]						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else cls_token
       snake_case   :							Optional[int]						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else unk_token
       snake_case   :							Optional[Any]						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       snake_case   :							Dict						  =AddedToken(lowercase_,    lstrip=lowercase_,    rstrip=lowercase_     ) if isinstance(lowercase_,    lowercase_     ) else mask_token

       super().__init__(
           errors=lowercase_,    bos_token=lowercase_,    eos_token=lowercase_,    unk_token=lowercase_,    sep_token=lowercase_,    cls_token=lowercase_,    pad_token=lowercase_,    mask_token=lowercase_,    add_prefix_space=lowercase_,    add_bos_token=lowercase_,    **lowercase_,    )
       snake_case   :							int						  =add_bos_token

       with open(lowercase_,    encoding='''utf-8'''     ) as vocab_handle:
          snake_case   :							List[str]						  =json.load(lowercase_     )
       snake_case   :							Any						  ={v: k for k, v in self.encoder.items()}
       snake_case   :							List[str]						  =errors  # how to handle errors in decoding
       snake_case   :							Union[str, Any]						  =bytes_to_unicode()
       snake_case   :							List[Any]						  ={v: k for k, v in self.byte_encoder.items()}
       with open(lowercase_,    encoding='''utf-8'''     ) as merges_handle:
          snake_case   :							Optional[Any]						  =merges_handle.read().split('''\n'''     )[1:-1]
       snake_case   :							Optional[Any]						  =[tuple(merge.split()     ) for merge in bpe_merges]
       snake_case   :							Tuple						  =dict(zip(lowercase_,    range(len(lowercase_     )     )     )     )
       snake_case   :							int						  ={}
       snake_case   :							List[str]						  =add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       snake_case   :							Tuple						  =re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''     )





    @property
    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
    def 					__snake_case       (    self    :    str     ):



       '''simple docstring'''







       return len(self.encoder     )





    def 					__snake_case       (    self    :    int     ):



       '''simple docstring'''







       return dict(self.encoder,    **self.added_tokens_encoder     )





    def 					__snake_case       (    self    :    List[str],    _snake_case    :    Optional[int]     ):



       '''simple docstring'''







       if token in self.cache:
          return self.cache[token]
       snake_case   :							Optional[int]						  =tuple(lowercase_     )
       snake_case   :							Optional[int]						  =get_pairs(lowercase_     )

       if not pairs:
          return token

       while True:
          snake_case   :							Any						  =min(lowercase_,    key=lambda _snake_case     : self.bpe_ranks.get(lowercase_,    float('''inf'''     )     )     )
          if bigram not in self.bpe_ranks:
             break
          snake_case   :							int						  =bigram
          snake_case   :							Any						  =[]
          snake_case   :							int						  =0
          while i < len(lowercase_     ):
             try:
                snake_case   :							Optional[Any]						  =word.index(lowercase_,    lowercase_     )
             except ValueError:
                new_word.extend(word[i:]     )
                break
             else:
                new_word.extend(word[i:j]     )
                snake_case   :							Optional[Any]						  =j

             if word[i] == first and i < len(lowercase_     ) - 1 and word[i + 1] == second:
                new_word.append(first + second     )
                i += 2
             else:
                new_word.append(word[i]     )
                i += 1
          snake_case   :							List[str]						  =tuple(lowercase_     )
          snake_case   :							Optional[Any]						  =new_word
          if len(lowercase_     ) == 1:
             break
          else:
             snake_case   :							List[Any]						  =get_pairs(lowercase_     )
       snake_case   :							Union[str, Any]						  =' '.join(lowercase_     )
       snake_case   :							Tuple						  =word
       return word





    def 					__snake_case       (    self    :    str,    _snake_case    :    List[int],    _snake_case    :    Optional[List[int]] = None     ):



       '''simple docstring'''







       if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       snake_case   :							List[str]						  =[self.cls_token_id]
       snake_case   :							Dict						  =[self.sep_token_id]
       return cls + token_ids_a + sep + token_ids_a + sep





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    List[int],    _snake_case    :    Optional[List[int]] = None,    _snake_case    :    bool = False     ):



       '''simple docstring'''







       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=lowercase_,    token_ids_a=lowercase_,    already_has_special_tokens=lowercase_     )

       if token_ids_a is None:
          return [1] + ([0] * len(lowercase_     )) + [1]
       return [1] + ([0] * len(lowercase_     )) + [1] + ([0] * len(lowercase_     )) + [1]





    def 					__snake_case       (    self    :    Optional[Any],    _snake_case    :    List[int],    _snake_case    :    Optional[List[int]] = None     ):



       '''simple docstring'''







       snake_case   :							str						  =[self.sep_token_id]
       snake_case   :							Optional[int]						  =[self.cls_token_id]

       if token_ids_a is None:
          return len(cls + token_ids_a + sep     ) * [0]
       return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    Any     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =[]
       for token in re.findall(self.pat,    lowercase_     ):
          snake_case   :							int						  =''.join(
              self.byte_encoder[b] for b in token.encode('''utf-8'''     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_     ).split(''' '''     )     )
       return bpe_tokens





    def 					__snake_case       (    self    :    int,    _snake_case    :    Any     ):



       '''simple docstring'''







       return self.encoder.get(lowercase_,    self.encoder.get(self.unk_token     )     )





    def 					__snake_case       (    self    :    Any,    _snake_case    :    Optional[Any]     ):



       '''simple docstring'''







       return self.decoder.get(lowercase_     )





    def 					__snake_case       (    self    :    Any,    _snake_case    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							int						  =''.join(lowercase_     )
       snake_case   :							int						  =bytearray([self.byte_decoder[c] for c in text]     ).decode('''utf-8''',    errors=self.errors     )
       return text





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    str,    _snake_case    :    Optional[str] = None     ):



       '''simple docstring'''







       if not os.path.isdir(lowercase_     ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
          return
       snake_case   :							Dict						  =os.path.join(
           lowercase_,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
       snake_case   :							Optional[int]						  =os.path.join(
           lowercase_,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

       with open(lowercase_,    '''w''',    encoding='''utf-8'''     ) as f:
          f.write(json.dumps(self.encoder,    indent=2,    sort_keys=lowercase_,    ensure_ascii=lowercase_     ) + '''\n'''     )

       snake_case   :							Union[str, Any]						  =0
       with open(lowercase_,    '''w''',    encoding='''utf-8'''     ) as writer:
          writer.write('''#version: 0.2\n'''     )
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),    key=lambda _snake_case     : kv[1]     ):
             if index != token_index:
                logger.warning(
                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                    ''' Please check that the tokenizer is not corrupted!'''     )
                snake_case   :							Dict						  =token_index
             writer.write(''' '''.join(lowercase_     ) + '''\n'''     )
             index += 1

       return vocab_file, merge_file





    def 					__snake_case       (    self    :    str,    _snake_case    :    Optional[int],    _snake_case    :    Optional[int]=False,    **_snake_case    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							Tuple						  =kwargs.pop('''add_prefix_space''',    self.add_prefix_space     )
       if (is_split_into_words or add_prefix_space) and (len(lowercase_     ) > 0 and not text[0].isspace()):
          snake_case   :							Tuple						  =' ' + text
       return (text, kwargs)






    def 					__snake_case       (    self    :    Dict,    _snake_case    :    "Conversation"     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =[]
       for is_user, text in conversation.iter_texts():
          input_ids.extend(self.encode(lowercase_,    add_special_tokens=lowercase_     ) + [self.eos_token_id]     )
       if len(lowercase_     ) > self.model_max_length:
          snake_case   :							str						  =input_ids[-self.model_max_length :]
       return input_ids




'''simple docstring'''





class 							lowerCAmelCase_		:





    def __init__(    self    :    List[Any],    _snake_case    :    Any     ):



       '''simple docstring'''







       snake_case   :							Tuple						  =data
       snake_case   :							int						  =None
       snake_case   :							List[Any]						  =None






    def __str__(    self    :    List[str]     ):



       '''simple docstring'''







       return f'''{self.data}'''





class 							lowerCAmelCase_		:





    def __init__(    self    :    Optional[int]     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =None
       snake_case   :							int						  =None





    def __iter__(    self    :    List[str]     ):



       '''simple docstring'''







       snake_case   :							str						  =self.head
       while node:
          yield node.data
          snake_case   :							Union[str, Any]						  =node.next





    def __str__(    self    :    List[str]     ):



       '''simple docstring'''







       return "->".join([str(_snake_case     ) for item in self]     )





    def __len__(    self    :    Dict     ):



       '''simple docstring'''







       return sum(1 for _ in self     )





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    str     ):



       '''simple docstring'''







       self.insert_at_nth(0,    _snake_case     )





    def 					__snake_case       (    self    :    Optional[Any],    _snake_case    :    Dict     ):



       '''simple docstring'''







       self.insert_at_nth(len(self     ),    _snake_case     )





    def 					__snake_case       (    self    :    Union[str, Any],    _snake_case    :    int,    _snake_case    :    Dict     ):



       '''simple docstring'''







       snake_case   :							Optional[int]						  =len(self     )

       if not 0 <= index <= length:
          raise IndexError('''list index out of range'''     )
       snake_case   :							int						  =Node(_snake_case     )
       if self.head is None:
          snake_case   :							Optional[Any]						  =new_node
       elif index == 0:
          snake_case   :							int						  =new_node
          snake_case   :							Union[str, Any]						  =self.head
          snake_case   :							Optional[int]						  =new_node
       elif index == length:
          snake_case   :							Dict						  =new_node
          snake_case   :							int						  =self.tail
          snake_case   :							Union[str, Any]						  =new_node
       else:
          snake_case   :							Optional[int]						  =self.head
          for _ in range(0,    _snake_case     ):
             snake_case   :							Union[str, Any]						  =temp.next
          snake_case   :							int						  =new_node
          snake_case   :							Union[str, Any]						  =temp.previous
          snake_case   :							List[Any]						  =temp
          snake_case   :							int						  =new_node





    def 					__snake_case       (    self    :    str     ):



       '''simple docstring'''







       return self.delete_at_nth(0     )





    def 					__snake_case       (    self    :    List[Any]     ):



       '''simple docstring'''







       return self.delete_at_nth(len(self     ) - 1     )





    def 					__snake_case       (    self    :    Union[str, Any],    _snake_case    :    int     ):



       '''simple docstring'''







       snake_case   :							Dict						  =len(self     )

       if not 0 <= index <= length - 1:
          raise IndexError('''list index out of range'''     )
       snake_case   :							List[str]						  =self.head  # default first node
       if length == 1:
          snake_case   :							Union[str, Any]						  =None
       elif index == 0:
          snake_case   :							Tuple						  =self.head.next
          snake_case   :							Optional[int]						  =None
       elif index == length - 1:
          snake_case   :							Union[str, Any]						  =self.tail
          snake_case   :							List[str]						  =self.tail.previous
          snake_case   :							int						  =None
       else:
          snake_case   :							int						  =self.head
          for _ in range(0,    _snake_case     ):
             snake_case   :							Any						  =temp.next
          snake_case   :							List[Any]						  =temp
          snake_case   :							Optional[Any]						  =temp.previous
          snake_case   :							int						  =temp.next
       return delete_node.data





    def 					__snake_case       (    self    :    List[Any],    _snake_case    :    Tuple     ):



       '''simple docstring'''







       snake_case   :							List[Any]						  =self.head

       while current.data != data:  # Find the position to delete
          if current.next:
             snake_case   :							Any						  =current.next
          else:  # We have reached the end an no value matches
             raise ValueError('''No data matching given value'''     )

       if current == self.head:
          self.delete_head()

       elif current == self.tail:
          self.delete_tail()

       else:  # Before: 1 <--> 2(current) <--> 3
          snake_case   :							Optional[int]						  =current.next  # 1 --> 3
          snake_case   :							List[Any]						  =current.previous  # 1 <--> 3
       return data






    def 					__snake_case       (    self    :    Any     ):



       '''simple docstring'''







       return len(self     ) == 0




def 		_a   (  ):
   snake_case   :							Optional[Any]						  =DoublyLinkedList()
   assert linked_list.is_empty() is True
   assert str(lowerCamelCase_							) == ""

   try:
      linked_list.delete_head()
      raise AssertionError  # This should not happen.
   except IndexError:
      assert True  # This should happen.

   try:
      linked_list.delete_tail()
      raise AssertionError  # This should not happen.
   except IndexError:
      assert True  # This should happen.

   for i in range(10							):
      assert len(lowerCamelCase_							) == i
      linked_list.insert_at_nth(lowerCamelCase_					,						i + 1							)
   assert str(lowerCamelCase_							) == "->".join(str(lowerCamelCase_							) for i in range(1					,						11							)							)

   linked_list.insert_at_head(0							)
   linked_list.insert_at_tail(11							)
   assert str(lowerCamelCase_							) == "->".join(str(lowerCamelCase_							) for i in range(0					,						12							)							)

   assert linked_list.delete_head() == 0
   assert linked_list.delete_at_nth(9							) == 10
   assert linked_list.delete_tail() == 11
   assert len(lowerCamelCase_							) == 9
   assert str(lowerCamelCase_							) == "->".join(str(lowerCamelCase_							) for i in range(1					,						10							)							)


if __name__ == "__main__":
      from doctest import testmod

      testmod()





