



'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

_UpperCamelCase							    =						logging.get_logger(__name__)

_UpperCamelCase							    =						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_UpperCamelCase							    =						{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

_UpperCamelCase							    =						{
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}

@lru_cache()
def 	a_					(							)				->				Tuple:
  __lowerCamelCase	:	int   	=			(
      list(range(ord('!'	)	,ord('~'	) + 1	)	) + list(range(ord('¡'	)	,ord('¬'	) + 1	)	) + list(range(ord('®'	)	,ord('ÿ'	) + 1	)	)
  )
  __lowerCamelCase	:	Optional[Any]   	=			bs[:]
  __lowerCamelCase	:	Union[str, Any]   	=			0
  for b in range(2**8	):
    if b not in bs:
      bs.append(_lowerCAmelCase	)
      cs.append(2**8 + n	)
      n += 1
  __lowerCamelCase	:	Optional[Any]   	=			[chr(_lowerCAmelCase	) for n in cs]
  return dict(zip(_lowerCAmelCase	,_lowerCAmelCase	)	)

def 	a_					(							_lowerCAmelCase	)				->				Optional[Any]:
  __lowerCamelCase	:	int   	=			set()
  __lowerCamelCase	:	str   	=			word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)	)
    __lowerCamelCase	:	List[Any]   	=			char
  return pairs






class  lowerCamelCase_   (    SCREAMING_SNAKE_CASE_					):
  """simple docstring"""




  a_							 =VOCAB_FILES_NAMES
  a_							 =PRETRAINED_VOCAB_FILES_MAP
  a_							 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							 =["""input_ids""", """attention_mask"""]




  def __init__(			self				:     int      ,     _a				:     Optional[int]      ,     _a				:     str      ,     _a				:     Union[str, Any]="replace"      ,     _a				:     int="<|endoftext|>"      ,     _a				:     Tuple="<|endoftext|>"      ,     _a				:     List[str]="<|endoftext|>"      ,     _a				:     List[Any]=None      ,     _a				:     Any=False      ,     _a				:     List[Any]=False      ,     **_a				:     Any      ,     ) -> Union[str, Any]:
    __lowerCamelCase	:	Dict   	=			AddedToken(_a      ,     lstrip=_a      ,     rstrip=_a       ) if isinstance(_a      ,     _a       ) else bos_token
    __lowerCamelCase	:	Optional[Any]   	=			AddedToken(_a      ,     lstrip=_a      ,     rstrip=_a       ) if isinstance(_a      ,     _a       ) else eos_token
    __lowerCamelCase	:	Tuple   	=			AddedToken(_a      ,     lstrip=_a      ,     rstrip=_a       ) if isinstance(_a      ,     _a       ) else unk_token
    __lowerCamelCase	:	int   	=			AddedToken(_a      ,     lstrip=_a      ,     rstrip=_a       ) if isinstance(_a      ,     _a       ) else pad_token
    super().__init__(
        errors=_a      ,     unk_token=_a      ,     bos_token=_a      ,     eos_token=_a      ,     pad_token=_a      ,     add_prefix_space=_a      ,     add_bos_token=_a      ,     **_a      ,     )
    __lowerCamelCase	:	str   	=			add_bos_token

    with open(_a      ,     encoding='utf-8'       ) as vocab_handle:
      __lowerCamelCase	:	int   	=			json.load(_a       )
    __lowerCamelCase	:	int   	=			{v: k for k, v in self.encoder.items()}
    __lowerCamelCase	:	List[Any]   	=			errors  # how to handle errors in decoding
    __lowerCamelCase	:	List[str]   	=			bytes_to_unicode()
    __lowerCamelCase	:	Tuple   	=			{v: k for k, v in self.byte_encoder.items()}
    with open(_a      ,     encoding='utf-8'       ) as merges_handle:
      __lowerCamelCase	:	Dict   	=			merges_handle.read().split('\n'       )[1:-1]
    __lowerCamelCase	:	int   	=			[tuple(merge.split()       ) for merge in bpe_merges]
    __lowerCamelCase	:	List[Any]   	=			dict(zip(_a      ,     range(len(_a       )       )       )       )
    __lowerCamelCase	:	Tuple   	=			{}
    __lowerCamelCase	:	Optional[Any]   	=			add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __lowerCamelCase	:	Dict   	=			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'       )




  @property
  def 						_lowercase					(			self				:     List[str]       ) -> Dict:
    return len(self.encoder       )




  def 						_lowercase					(			self				:     Tuple       ) -> Tuple:
    return dict(self.encoder      ,     **self.added_tokens_encoder       )




  def 						_lowercase					(			self				:     Any      ,     _a				:     int       ) -> Optional[Any]:
    if token in self.cache:
      return self.cache[token]
    __lowerCamelCase	:	str   	=			tuple(_a       )
    __lowerCamelCase	:	List[str]   	=			get_pairs(_a       )

    if not pairs:
      return token

    while True:
      __lowerCamelCase	:	Tuple   	=			min(_a      ,     key=lambda _a       : self.bpe_ranks.get(_a      ,     float('inf'       )       )       )
      if bigram not in self.bpe_ranks:
        break
      __lowerCamelCase	,__lowerCamelCase	:	Optional[Any]   	=			bigram
      __lowerCamelCase	:	Optional[int]   	=			[]
      __lowerCamelCase	:	List[str]   	=			0
      while i < len(_a       ):
        try:
          __lowerCamelCase	:	Any   	=			word.index(_a      ,     _a       )
        except ValueError:
          new_word.extend(word[i:]       )
          break
        else:
          new_word.extend(word[i:j]       )
          __lowerCamelCase	:	Optional[int]   	=			j

        if word[i] == first and i < len(_a       ) - 1 and word[i + 1] == second:
          new_word.append(first + second       )
          i += 2
        else:
          new_word.append(word[i]       )
          i += 1
      __lowerCamelCase	:	List[str]   	=			tuple(_a       )
      __lowerCamelCase	:	int   	=			new_word
      if len(_a       ) == 1:
        break
      else:
        __lowerCamelCase	:	str   	=			get_pairs(_a       )
    __lowerCamelCase	:	List[Any]   	=			' '.join(_a       )
    __lowerCamelCase	:	Optional[Any]   	=			word
    return word




  def 						_lowercase					(			self				:     int      ,     _a				:     Optional[int]      ,     _a				:     Dict=None       ) -> Optional[int]:
    if self.add_bos_token:
      __lowerCamelCase	:	List[str]   	=			[self.bos_token_id]
    else:
      __lowerCamelCase	:	Union[str, Any]   	=			[]

    __lowerCamelCase	:	str   	=			bos_token_ids + token_ids_a

    if token_ids_a is None:
      return output

    return output + bos_token_ids + token_ids_a




  def 						_lowercase					(			self				:     str      ,     _a				:     List[int]      ,     _a				:     Optional[List[int]] = None      ,     _a				:     bool = False       ) -> List[int]:
    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=_a      ,     token_ids_a=_a      ,     already_has_special_tokens=_a       )

    if not self.add_bos_token:
      return super().get_special_tokens_mask(
          token_ids_a=_a      ,     token_ids_a=_a      ,     already_has_special_tokens=_a       )

    if token_ids_a is None:
      return [1] + ([0] * len(_a       ))
    return [1] + ([0] * len(_a       )) + [1] + ([0] * len(_a       ))




  def 						_lowercase					(			self				:     str      ,     _a				:     Dict       ) -> str:
    __lowerCamelCase	:	Tuple   	=			[]
    for token in re.findall(self.pat      ,     _a       ):
      __lowerCamelCase	:	Union[str, Any]   	=			''.join(
          self.byte_encoder[b] for b in token.encode('utf-8'       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
      bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_a       ).split(' '       )       )
    return bpe_tokens




  def 						_lowercase					(			self				:     List[str]      ,     _a				:     List[str]       ) -> int:
    return self.encoder.get(_a      ,     self.encoder.get(self.unk_token       )       )




  def 						_lowercase					(			self				:     Dict      ,     _a				:     Union[str, Any]       ) -> List[Any]:
    return self.decoder.get(_a       )




  def 						_lowercase					(			self				:     Dict      ,     _a				:     Any       ) -> str:
    __lowerCamelCase	:	List[str]   	=			''.join(_a       )
    __lowerCamelCase	:	List[str]   	=			bytearray([self.byte_decoder[c] for c in text]       ).decode('utf-8'      ,     errors=self.errors       )
    return text




  def 						_lowercase					(			self				:     int      ,     _a				:     str      ,     _a				:     Optional[str] = None       ) -> Tuple[str]:
    if not os.path.isdir(_a       ):
      logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
      return
    __lowerCamelCase	:	Any   	=			os.path.join(
        _a      ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
    __lowerCamelCase	:	str   	=			os.path.join(
        _a      ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']       )

    with open(_a      ,     'w'      ,     encoding='utf-8'       ) as f:
      f.write(json.dumps(self.encoder      ,     indent=2      ,     sort_keys=_a      ,     ensure_ascii=_a       ) + '\n'       )

    __lowerCamelCase	:	Tuple   	=			0
    with open(_a      ,     'w'      ,     encoding='utf-8'       ) as writer:
      writer.write('#version: 0.2\n'       )
      for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,     key=lambda _a       : kv[1]       ):
        if index != token_index:
          logger.warning(
              f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
              ' Please check that the tokenizer is not corrupted!'       )
          __lowerCamelCase	:	List[Any]   	=			token_index
        writer.write(' '.join(_a       ) + '\n'       )
        index += 1

    return vocab_file, merge_file




  def 						_lowercase					(			self				:     Optional[int]      ,     _a				:     str      ,     _a				:     int=False      ,     **_a				:     str       ) -> str:
    __lowerCamelCase	:	Dict   	=			kwargs.pop('add_prefix_space'      ,     self.add_prefix_space       )
    if is_split_into_words or add_prefix_space:
      __lowerCamelCase	:	List[str]   	=			' ' + text
    return (text, kwargs)




  def 						_lowercase					(			self				:     Optional[Any]      ,     _a				:     "Conversation"       ) -> List[int]:
    __lowerCamelCase	:	List[str]   	=			[]
    for is_user, text in conversation.iter_texts():
      input_ids.extend(self.encode(_a      ,     add_special_tokens=_a       ) + [self.eos_token_id]       )
    if len(_a       ) > self.model_max_length:
      __lowerCamelCase	:	Any   	=			input_ids[-self.model_max_length :]
    return input_ids











'''simple docstring'''





import math
from timeit import timeit

def 	a_					(							_lowerCAmelCase	)				->				int:
  __lowerCamelCase	:	int   	=			0
  __lowerCamelCase	:	Optional[int]   	=			abs(_lowerCAmelCase	)
  while True:
    __lowerCamelCase	:	List[str]   	=			n // 10
    digits += 1
    if n == 0:
      break
  return digits

def 	a_					(							_lowerCAmelCase	)				->				int:
  return 1 if n == 0 else math.floor(math.log(abs(_lowerCAmelCase	)	,10	) + 1	)

def 	a_					(							_lowerCAmelCase	)				->				int:
  return len(str(abs(_lowerCAmelCase	)	)	)

def 	a_					(							)				->				None:
  from collections.abc import Callable

  def benchmark_a_function(_lowerCAmelCase	,_lowerCAmelCase	) -> None:
    __lowerCamelCase	:	Any   	=			F'{func.__name__}({value})'
    __lowerCamelCase	:	int   	=			timeit(F'__main__.{call}'	,setup='import __main__'	)
    print(F'{call}: {func(_lowerCAmelCase	)} -- {timing} seconds'	)

  for value in (262144, 1125899906842624, 1267650600228229401496703205376):
    for func in (num_digits, num_digits_fast, num_digits_faster):
      benchmark_a_function(_lowerCAmelCase	,_lowerCAmelCase	)
    print()


if __name__ == "__main__":
   import doctest

   doctest.testmod()
   benchmark()






