


'''simple docstring'''

import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

lowerCamelCase							:       Dict  						=     logging.get_logger(__name__)

lowerCamelCase							:       List[str]  						=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

lowerCamelCase							:       List[Any]  						=     {
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

lowerCamelCase							:       Tuple  						=     {
    'microsoft/deberta-base': 512,
    'microsoft/deberta-large': 512,
    'microsoft/deberta-xlarge': 512,
    'microsoft/deberta-base-mnli': 512,
    'microsoft/deberta-large-mnli': 512,
    'microsoft/deberta-xlarge-mnli': 512,
}

lowerCamelCase							:       Union[str, Any]  						=     {
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}

def _SCREAMING_SNAKE_CASE				()	->	Tuple:






       """simple docstring"""







       lowercase__				      =			(
           list(range(ord('''!'''	)  ,			ord('''~'''	) + 1	)	) + list(range(ord('''¡'''	)  ,			ord('''¬'''	) + 1	)	) + list(range(ord('''®'''	)  ,			ord('''ÿ'''	) + 1	)	)
       )
       lowercase__				      =			bs[:]
       lowercase__				      =			0
       for b in range(2**8	):
              if b not in bs:
                     bs.append(A	)
                     cs.append(2**8 + n	)
                     n += 1
       lowercase__				      =			[chr(A	) for n in cs]
       return dict(zip(A  ,			A	)	)

def _SCREAMING_SNAKE_CASE				(A	)	->	Optional[int]:






       """simple docstring"""







       lowercase__				      =			set()
       lowercase__				      =			word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)	)
              lowercase__				      =			char
       return pairs

class   __lowerCAmelCase						(SCREAMING_SNAKE_CASE_					):





      '''simple docstring'''






      lowerCAmelCase__ :					Dict   =							VOCAB_FILES_NAMES
      lowerCAmelCase__ :					Any   =							PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__ :					int   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__ :					Dict   =							["input_ids", "attention_mask", "token_type_ids"]



      def __init__(self   :   Dict       ,						UpperCamelCase   :   Dict       ,						UpperCamelCase   :   Any       ,						UpperCamelCase   :   Any="replace"       ,						UpperCamelCase   :   Any="[CLS]"       ,						UpperCamelCase   :   Union[str, Any]="[SEP]"       ,						UpperCamelCase   :   str="[SEP]"       ,						UpperCamelCase   :   Dict="[CLS]"       ,						UpperCamelCase   :   Optional[int]="[UNK]"       ,						UpperCamelCase   :   int="[PAD]"       ,						UpperCamelCase   :   str="[MASK]"       ,						UpperCamelCase   :   str=False       ,						UpperCamelCase   :   Tuple=False       ,						**UpperCamelCase   :   Tuple       ,						):

             '''simple docstring'''







             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else bos_token
             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else eos_token
             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else sep_token
             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else cls_token
             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else unk_token
             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             lowercase__				      =			AddedToken(lowercase_       ,						lstrip=lowercase_       ,						rstrip=lowercase_			) if isinstance(lowercase_       ,						lowercase_			) else mask_token

             super().__init__(
                 errors=lowercase_       ,						bos_token=lowercase_       ,						eos_token=lowercase_       ,						unk_token=lowercase_       ,						sep_token=lowercase_       ,						cls_token=lowercase_       ,						pad_token=lowercase_       ,						mask_token=lowercase_       ,						add_prefix_space=lowercase_       ,						add_bos_token=lowercase_       ,						**lowercase_       ,						)
             lowercase__				      =			add_bos_token

             with open(lowercase_       ,						encoding='''utf-8'''			) as vocab_handle:
                    lowercase__				      =			json.load(lowercase_			)
             lowercase__				      =			{v: k for k, v in self.encoder.items()}
             lowercase__				      =			errors  # how to handle errors in decoding
             lowercase__				      =			bytes_to_unicode()
             lowercase__				      =			{v: k for k, v in self.byte_encoder.items()}
             with open(lowercase_       ,						encoding='''utf-8'''			) as merges_handle:
                    lowercase__				      =			merges_handle.read().split('''\n'''			)[1:-1]
             lowercase__				      =			[tuple(merge.split()			) for merge in bpe_merges]
             lowercase__				      =			dict(zip(lowercase_       ,						range(len(lowercase_			)			)			)			)
             lowercase__				      =			{}
             lowercase__				      =			add_prefix_space

             # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
             lowercase__				      =			re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''			)



      @property
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
      def 				UpperCamelCase__		(self   :   int			):

             '''simple docstring'''







             return len(self.encoder			)



      def 				UpperCamelCase__		(self   :   int			):

             '''simple docstring'''







             return dict(self.encoder       ,						**self.added_tokens_encoder			)



      def 				UpperCamelCase__		(self   :   Optional[int]       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             if token in self.cache:
                    return self.cache[token]
             lowercase__				      =			tuple(lowercase_			)
             lowercase__				      =			get_pairs(lowercase_			)

             if not pairs:
                    return token

             while True:
                    lowercase__				      =			min(lowercase_       ,						key=lambda UpperCamelCase			: self.bpe_ranks.get(lowercase_       ,						float('''inf'''			)			)			)
                    if bigram not in self.bpe_ranks:
                           break
                    lowercase__      ,lowercase__				      =			bigram
                    lowercase__				      =			[]
                    lowercase__				      =			0
                    while i < len(lowercase_			):
                           try:
                                  lowercase__				      =			word.index(lowercase_       ,						lowercase_			)
                           except ValueError:
                                  new_word.extend(word[i:]			)
                                  break
                           else:
                                  new_word.extend(word[i:j]			)
                                  lowercase__				      =			j

                           if word[i] == first and i < len(lowercase_			) - 1 and word[i + 1] == second:
                                  new_word.append(first + second			)
                                  i += 2
                           else:
                                  new_word.append(word[i]			)
                                  i += 1
                    lowercase__				      =			tuple(lowercase_			)
                    lowercase__				      =			new_word
                    if len(lowercase_			) == 1:
                           break
                    else:
                           lowercase__				      =			get_pairs(lowercase_			)
             lowercase__				      =			''' '''.join(lowercase_			)
             lowercase__				      =			word
             return word



      def 				UpperCamelCase__		(self   :   int       ,						UpperCamelCase   :   Any       ,						UpperCamelCase   :   str = None			):

             '''simple docstring'''







             if token_ids_a is None:
                    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             lowercase__				      =			[self.cls_token_id]
             lowercase__				      =			[self.sep_token_id]
             return cls + token_ids_a + sep + token_ids_a + sep



      def 				UpperCamelCase__		(self   :   int       ,						UpperCamelCase   :   Tuple       ,						UpperCamelCase   :   Optional[int] = None       ,						UpperCamelCase   :   Dict = False			):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=lowercase_       ,						token_ids_a=lowercase_       ,						already_has_special_tokens=lowercase_			)

             if token_ids_a is None:
                    return [1] + ([0] * len(lowercase_			)) + [1]
             return [1] + ([0] * len(lowercase_			)) + [1] + ([0] * len(lowercase_			)) + [1]



      def 				UpperCamelCase__		(self   :   int       ,						UpperCamelCase   :   List[str]       ,						UpperCamelCase   :   int = None			):

             '''simple docstring'''







             lowercase__				      =			[self.sep_token_id]
             lowercase__				      =			[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep			) * [0]
             return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]



      def 				UpperCamelCase__		(self   :   Any       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             lowercase__				      =			[]
             for token in re.findall(self.pat       ,						lowercase_			):
                    lowercase__				      =			''''''.join(
                        self.byte_encoder[b] for b in token.encode('''utf-8'''			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                    bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_			).split(''' '''			)			)
             return bpe_tokens



      def 				UpperCamelCase__		(self   :   int       ,						UpperCamelCase   :   Tuple			):

             '''simple docstring'''







             return self.encoder.get(lowercase_       ,						self.encoder.get(self.unk_token			)			)



      def 				UpperCamelCase__		(self   :   List[str]       ,						UpperCamelCase   :   Optional[int]			):

             '''simple docstring'''







             return self.decoder.get(lowercase_			)



      def 				UpperCamelCase__		(self   :   Optional[int]       ,						UpperCamelCase   :   Optional[Any]			):

             '''simple docstring'''







             lowercase__				      =			''''''.join(lowercase_			)
             lowercase__				      =			bytearray([self.byte_decoder[c] for c in text]			).decode('''utf-8'''       ,						errors=self.errors			)
             return text



      def 				UpperCamelCase__		(self   :   Dict       ,						UpperCamelCase   :   List[Any]       ,						UpperCamelCase   :   int = None			):

             '''simple docstring'''







             if not os.path.isdir(lowercase_			):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
                    return
             lowercase__				      =			os.path.join(
                 lowercase_       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
             lowercase__				      =			os.path.join(
                 lowercase_       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

             with open(lowercase_       ,						'''w'''       ,						encoding='''utf-8'''			) as f:
                    f.write(json.dumps(self.encoder       ,						indent=2       ,						sort_keys=lowercase_       ,						ensure_ascii=lowercase_			) + '''\n'''			)

             lowercase__				      =			0
             with open(lowercase_       ,						'''w'''       ,						encoding='''utf-8'''			) as writer:
                    writer.write('''#version: 0.2\n'''			)
                    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,						key=lambda UpperCamelCase			: kv[1]			):
                           if index != token_index:
                                  logger.warning(
                                      f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                      ''' Please check that the tokenizer is not corrupted!'''			)
                                  lowercase__				      =			token_index
                           writer.write(''' '''.join(lowercase_			) + '''\n'''			)
                           index += 1

             return vocab_file, merge_file



      def 				UpperCamelCase__		(self   :   Dict       ,						UpperCamelCase   :   Union[str, Any]       ,						UpperCamelCase   :   Union[str, Any]=False       ,						**UpperCamelCase   :   int			):

             '''simple docstring'''







             lowercase__				      =			kwargs.pop('''add_prefix_space'''       ,						self.add_prefix_space			)
             if (is_split_into_words or add_prefix_space) and (len(lowercase_			) > 0 and not text[0].isspace()):
                    lowercase__				      =			''' ''' + text
             return (text, kwargs)






      def 				UpperCamelCase__		(self   :   Tuple       ,						UpperCamelCase   :   List[str]			):

             '''simple docstring'''







             lowercase__				      =			[]
             for is_user, text in conversation.iter_texts():
                    input_ids.extend(self.encode(lowercase_       ,						add_special_tokens=lowercase_			) + [self.eos_token_id]			)
             if len(lowercase_			) > self.model_max_length:
                    lowercase__				      =			input_ids[-self.model_max_length :]
             return input_ids








'''simple docstring'''

def _SCREAMING_SNAKE_CASE				(A	)	->	int:






       """simple docstring"""







       return 1 if input_a == 0 else 0

def _SCREAMING_SNAKE_CASE				()	->	None:






       """simple docstring"""







       assert not_gate(0	) == 1
       assert not_gate(1	) == 0


if __name__ == "__main__":
 print(not_gate(0))
 print(not_gate(1))




