





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

a__							:							List[Any]			=logging.get_logger(__name__)

a__							:							Dict			={
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

a__							:							List[str]			={
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

a__							:							Tuple			={
    '''gpt2''': 1_024,
    '''gpt2-medium''': 1_024,
    '''gpt2-large''': 1_024,
    '''gpt2-xl''': 1_024,
    '''distilgpt2''': 1_024,
}
@lru_cache()
def     lowercase__       (       )			->				str:



       """simple docstring"""






       __UpperCamelCase	=  (
           list(range(ord('!'					)    ,  ord('~'					) + 1					)					) + list(range(ord('¡'					)    ,  ord('¬'					) + 1					)					) + list(range(ord('®'					)    ,  ord('ÿ'					) + 1					)					)
       )
       __UpperCamelCase	=  bs[:]
       __UpperCamelCase	=  0
       for b in range(2**8					):
              if b not in bs:
                     bs.append(__lowercase					)
                     cs.append(2**8 + n					)
                     n += 1
       __UpperCamelCase	=  [chr(__lowercase					) for n in cs]
       return dict(zip(__lowercase    ,  __lowercase					)					)
def     lowercase__       (       __lowercase						:  List[str]					)			->				List[Any]:



       """simple docstring"""






       __UpperCamelCase	=  set()
       __UpperCamelCase	=  word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              __UpperCamelCase	=  char
       return pairs


class 					snake_case					(   __lowerCamelCase	):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_     : int      		=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_     : Optional[int]      		=PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_     : Optional[Any]      		=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_     : Union[str, Any]      		=["input_ids", "attention_mask"]






 def __init__(	self	:							int							,     __A	:							Optional[int]							,     __A	:							Union[str, Any]							,     __A	:							Any="replace"							,     __A	:							Optional[Any]="<|endoftext|>"							,     __A	:							Union[str, Any]="<|endoftext|>"							,     __A	:							List[Any]="<|endoftext|>"							,     __A	:							Optional[int]=None							,     __A	:							List[Any]=False							,     __A	:							int=False							,     **__A	:							List[str]							,     ):
        __UpperCamelCase	=  AddedToken(__A							,     lstrip=__A							,     rstrip=__A     ) if isinstance(__A							,     __A     ) else bos_token
        __UpperCamelCase	=  AddedToken(__A							,     lstrip=__A							,     rstrip=__A     ) if isinstance(__A							,     __A     ) else eos_token
        __UpperCamelCase	=  AddedToken(__A							,     lstrip=__A							,     rstrip=__A     ) if isinstance(__A							,     __A     ) else unk_token
        __UpperCamelCase	=  AddedToken(__A							,     lstrip=__A							,     rstrip=__A     ) if isinstance(__A							,     __A     ) else pad_token
        super().__init__(
            errors=__A							,     unk_token=__A							,     bos_token=__A							,     eos_token=__A							,     pad_token=__A							,     add_prefix_space=__A							,     add_bos_token=__A							,     **__A							,     )
        __UpperCamelCase	=  add_bos_token

        with open(__A							,     encoding='utf-8'     ) as vocab_handle:
               __UpperCamelCase	=  json.load(__A     )
        __UpperCamelCase	=  {v: k for k, v in self.encoder.items()}
        __UpperCamelCase	=  errors  # how to handle errors in decoding
        __UpperCamelCase	=  bytes_to_unicode()
        __UpperCamelCase	=  {v: k for k, v in self.byte_encoder.items()}
        with open(__A							,     encoding='utf-8'     ) as merges_handle:
               __UpperCamelCase	=  merges_handle.read().split('\n'     )[1:-1]
        __UpperCamelCase	=  [tuple(merge.split()     ) for merge in bpe_merges]
        __UpperCamelCase	=  dict(zip(__A							,     range(len(__A     )     )     )     )
        __UpperCamelCase	=  {}
        __UpperCamelCase	=  add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        __UpperCamelCase	=  re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )






 @property
 def 	_lowerCamelCase (	self	:							Any     ):
        return len(self.encoder     )






 def 	_lowerCamelCase (	self	:							Any     ):
        return dict(self.encoder							,     **self.added_tokens_encoder     )






 def 	_lowerCamelCase (	self	:							Optional[Any]							,     __A	:							str     ):
        if token in self.cache:
               return self.cache[token]
        __UpperCamelCase	=  tuple(__A     )
        __UpperCamelCase	=  get_pairs(__A     )

        if not pairs:
               return token

        while True:
               __UpperCamelCase	=  min(__A							,     key=lambda __A     : self.bpe_ranks.get(__A							,     float('inf'     )     )     )
               if bigram not in self.bpe_ranks:
                      break
               __UpperCamelCase	,						__UpperCamelCase	=  bigram
               __UpperCamelCase	=  []
               __UpperCamelCase	=  0
               while i < len(__A     ):
                      try:
                             __UpperCamelCase	=  word.index(__A							,     __A     )
                      except ValueError:
                             new_word.extend(word[i:]     )
                             break
                      else:
                             new_word.extend(word[i:j]     )
                             __UpperCamelCase	=  j

                      if word[i] == first and i < len(__A     ) - 1 and word[i + 1] == second:
                             new_word.append(first + second     )
                             i += 2
                      else:
                             new_word.append(word[i]     )
                             i += 1
               __UpperCamelCase	=  tuple(__A     )
               __UpperCamelCase	=  new_word
               if len(__A     ) == 1:
                      break
               else:
                      __UpperCamelCase	=  get_pairs(__A     )
        __UpperCamelCase	=  ' '.join(__A     )
        __UpperCamelCase	=  word
        return word






 def 	_lowerCamelCase (	self	:							Tuple							,     __A	:							Union[str, Any]							,     __A	:							str=None     ):
        if self.add_bos_token:
               __UpperCamelCase	=  [self.bos_token_id]
        else:
               __UpperCamelCase	=  []

        __UpperCamelCase	=  bos_token_ids + token_ids_a

        if token_ids_a is None:
               return output

        return output + bos_token_ids + token_ids_a






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							List[int]							,     __A	:							Optional[List[int]] = None							,     __A	:							bool = False     ):
        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__A							,     token_ids_a=__A							,     already_has_special_tokens=__A     )

        if not self.add_bos_token:
               return super().get_special_tokens_mask(
                   token_ids_a=__A							,     token_ids_a=__A							,     already_has_special_tokens=__A     )

        if token_ids_a is None:
               return [1] + ([0] * len(__A     ))
        return [1] + ([0] * len(__A     )) + [1] + ([0] * len(__A     ))






 def 	_lowerCamelCase (	self	:							int							,     __A	:							Dict     ):
        __UpperCamelCase	=  []
        for token in re.findall(self.pat							,     __A     ):
               __UpperCamelCase	=  ''.join(
                   self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A     ).split(' '     )     )
        return bpe_tokens






 def 	_lowerCamelCase (	self	:							Optional[int]							,     __A	:							Dict     ):
        return self.encoder.get(__A							,     self.encoder.get(self.unk_token     )     )






 def 	_lowerCamelCase (	self	:							str							,     __A	:							Optional[int]     ):
        return self.decoder.get(__A     )






 def 	_lowerCamelCase (	self	:							Optional[int]							,     __A	:							List[Any]     ):
        __UpperCamelCase	=  ''.join(__A     )
        __UpperCamelCase	=  bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8'							,     errors=self.errors     )
        return text






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							str							,     __A	:							Optional[str] = None     ):
        if not os.path.isdir(__A     ):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
               return
        __UpperCamelCase	=  os.path.join(
            __A							,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
        __UpperCamelCase	=  os.path.join(
            __A							,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

        with open(__A							,     'w'							,     encoding='utf-8'     ) as f:
               f.write(json.dumps(self.encoder							,     indent=2							,     sort_keys=__A							,     ensure_ascii=__A     ) + '\n'     )

        __UpperCamelCase	=  0
        with open(__A							,     'w'							,     encoding='utf-8'     ) as writer:
               writer.write('#version: 0.2\n'     )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,     key=lambda __A     : kv[1]     ):
                      if index != token_index:
                             logger.warning(
                                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                 ' Please check that the tokenizer is not corrupted!'     )
                             __UpperCamelCase	=  token_index
                      writer.write(' '.join(__A     ) + '\n'     )
                      index += 1

        return vocab_file, merge_file






 def 	_lowerCamelCase (	self	:							Tuple							,     __A	:							List[str]							,     __A	:							Optional[int]=False							,     **__A	:							List[Any]     ):
        __UpperCamelCase	=  kwargs.pop('add_prefix_space'							,     self.add_prefix_space     )
        if is_split_into_words or add_prefix_space:
               __UpperCamelCase	=  ' ' + text
        return (text, kwargs)






 def 	_lowerCamelCase (	self	:							List[str]							,     __A	:							"Conversation"     ):
        __UpperCamelCase	=  []
        for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(__A							,     add_special_tokens=__A     ) + [self.eos_token_id]     )
        if len(__A     ) > self.model_max_length:
               __UpperCamelCase	=  input_ids[-self.model_max_length :]
        return input_ids







'''simple docstring'''
def     lowercase__       (       __lowercase						:  list    ,  __lowercase						:  int    ,  __lowercase						:  int    ,  __lowercase						:  int					)			->				int:



       """simple docstring"""






       __UpperCamelCase	=  int((lower_bound + upper_bound) // 2					)
       if array[r] == value:
              return r
       if lower_bound >= upper_bound:
              return -1
       if array[r] < value:
              return binary_search(__lowercase    ,  r + 1    ,  __lowercase    ,  __lowercase					)
       else:
              return binary_search(__lowercase    ,  __lowercase    ,  r - 1    ,  __lowercase					)
def     lowercase__       (       __lowercase						:  int    ,  __lowercase						:  list					)			->				list:



       """simple docstring"""






       __UpperCamelCase	=  0
       if matrix[index][0] == value:
              return [index, 0]
       while index < len(__lowercase					) and matrix[index][0] < value:
              __UpperCamelCase	=  binary_search(matrix[index]    ,  0    ,  len(matrix[index]					) - 1    ,  __lowercase					)
              if r != -1:
                     return [index, r]
              index += 1
       return [-1, -1]


if __name__ == "__main__":
    import doctest

    doctest.testmod()
