




'''simple docstring'''






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase				     = logging.get_logger(__name__)

_lowerCAmelCase				     = {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCAmelCase				     = {
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
}

_lowerCAmelCase				     = {
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}

@lru_cache()
def       __lowerCAmelCase (   ):
			__UpperCamelCase				:      Optional[int]									=       (
			    list(range(ord("!"						) ,				ord("~"						) + 1						)						) + list(range(ord("¡"						) ,				ord("¬"						) + 1						)						) + list(range(ord("®"						) ,				ord("ÿ"						) + 1						)						)
			)
			__UpperCamelCase				:      List[str]									=       bs[:]
			__UpperCamelCase				:      List[Any]									=       0
			for b in range(2**8						):
						if b not in bs:
									bs.append(snake_case__						)
									cs.append(2**8 + n						)
									n += 1
			__UpperCamelCase				:      Tuple									=       [chr(snake_case__						) for n in cs]
			return dict(zip(snake_case__ ,				snake_case__						)						)

def       __lowerCAmelCase (   snake_case__						):
			__UpperCamelCase				:      int									=       set()
			__UpperCamelCase				:      Dict									=       word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)						)
						__UpperCamelCase				:      Optional[int]									=       char
			return pairs




class  A		(			SCREAMING_SNAKE_CASE__		):




				'''simple docstring'''





				A				     =    VOCAB_FILES_NAMES
				A				     =    PRETRAINED_VOCAB_FILES_MAP
				A				     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A				     =    ["input_ids", "attention_mask"]
				def __init__(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase="replace"			,  _UpperCAmelCase="<s>"			,  _UpperCAmelCase="</s>"			,  _UpperCAmelCase="</s>"			,  _UpperCAmelCase="<s>"			,  _UpperCAmelCase="<unk>"			,  _UpperCAmelCase="<pad>"			,  _UpperCAmelCase="<mask>"			,  _UpperCAmelCase=False			,  **_UpperCAmelCase			,  )     ->	str:
							__UpperCamelCase				:      List[Any]									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else bos_token
							__UpperCamelCase				:      Union[str, Any]									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else eos_token
							__UpperCamelCase				:      Optional[Any]									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else sep_token
							__UpperCamelCase				:      Dict									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else cls_token
							__UpperCamelCase				:      Optional[int]									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else unk_token
							__UpperCamelCase				:      Dict									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							__UpperCamelCase				:      Dict									=       AddedToken(_UpperCAmelCase			,  lstrip=_UpperCAmelCase			,  rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase			,  _UpperCAmelCase      ) else mask_token

							super().__init__(
							    errors=_UpperCAmelCase			,  bos_token=_UpperCAmelCase			,  eos_token=_UpperCAmelCase			,  unk_token=_UpperCAmelCase			,  sep_token=_UpperCAmelCase			,  cls_token=_UpperCAmelCase			,  pad_token=_UpperCAmelCase			,  mask_token=_UpperCAmelCase			,  add_prefix_space=_UpperCAmelCase			,  **_UpperCAmelCase			,  )

							with open(_UpperCAmelCase			,  encoding="utf-8"      ) as vocab_handle:
										__UpperCamelCase				:      str									=       json.load(_UpperCAmelCase      )
							__UpperCamelCase				:      str									=       {v: k for k, v in self.encoder.items()}
							__UpperCamelCase				:      Dict									=       errors  # how to handle errors in decoding
							__UpperCamelCase				:      int									=       bytes_to_unicode()
							__UpperCamelCase				:      Optional[Any]									=       {v: k for k, v in self.byte_encoder.items()}
							with open(_UpperCAmelCase			,  encoding="utf-8"      ) as merges_handle:
										__UpperCamelCase				:      Optional[int]									=       merges_handle.read().split("\n"      )[1:-1]
							__UpperCamelCase				:      Tuple									=       [tuple(merge.split()      ) for merge in bpe_merges]
							__UpperCamelCase				:      Tuple									=       dict(zip(_UpperCAmelCase			,  range(len(_UpperCAmelCase      )      )      )      )
							__UpperCamelCase				:      List[str]									=       {}
							__UpperCamelCase				:      Optional[Any]									=       add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							__UpperCamelCase				:      Any									=       re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"      )
				@property
				def 	a_	(self      )     ->	List[str]:
							return len(self.encoder      )
				def 	a_	(self      )     ->	List[Any]:
							return dict(self.encoder			,  **self.added_tokens_encoder      )
				def 	a_	(self			,  _UpperCAmelCase      )     ->	str:
							if token in self.cache:
										return self.cache[token]
							__UpperCamelCase				:      Any									=       tuple(_UpperCAmelCase      )
							__UpperCamelCase				:      Tuple									=       get_pairs(_UpperCAmelCase      )

							if not pairs:
										return token

							while True:
										__UpperCamelCase				:      Any									=       min(_UpperCAmelCase			,  key=lambda _UpperCAmelCase      : self.bpe_ranks.get(_UpperCAmelCase			,  float("inf"      )      )      )
										if bigram not in self.bpe_ranks:
													break
										__UpperCamelCase		, __UpperCamelCase				:      Optional[Any]									=       bigram
										__UpperCamelCase				:      Dict									=       []
										__UpperCamelCase				:      str									=       0
										while i < len(_UpperCAmelCase      ):
													try:
																__UpperCamelCase				:      Optional[Any]									=       word.index(_UpperCAmelCase			,  _UpperCAmelCase      )
													except ValueError:
																new_word.extend(word[i:]      )
																break
													else:
																new_word.extend(word[i:j]      )
																__UpperCamelCase				:      List[str]									=       j

													if word[i] == first and i < len(_UpperCAmelCase      ) - 1 and word[i + 1] == second:
																new_word.append(first + second      )
																i += 2
													else:
																new_word.append(word[i]      )
																i += 1
										__UpperCamelCase				:      List[str]									=       tuple(_UpperCAmelCase      )
										__UpperCamelCase				:      Optional[int]									=       new_word
										if len(_UpperCAmelCase      ) == 1:
													break
										else:
													__UpperCamelCase				:      List[Any]									=       get_pairs(_UpperCAmelCase      )
							__UpperCamelCase				:      Optional[Any]									=       " ".join(_UpperCAmelCase      )
							__UpperCamelCase				:      Optional[int]									=       word
							return word
				def 	a_	(self			,  _UpperCAmelCase      )     ->	str:
							__UpperCamelCase				:      Dict									=       []
							for token in re.findall(self.pat			,  _UpperCAmelCase      ):
										__UpperCamelCase				:      Tuple									=       "".join(
										    self.byte_encoder[b] for b in token.encode("utf-8"      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
										bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCAmelCase      ).split(" "      )      )
							return bpe_tokens
				def 	a_	(self			,  _UpperCAmelCase      )     ->	Optional[Any]:
							return self.encoder.get(_UpperCAmelCase			,  self.encoder.get(self.unk_token      )      )
				def 	a_	(self			,  _UpperCAmelCase      )     ->	List[Any]:
							return self.decoder.get(_UpperCAmelCase      )
				def 	a_	(self			,  _UpperCAmelCase      )     ->	List[Any]:
							__UpperCamelCase				:      Optional[int]									=       "".join(_UpperCAmelCase      )
							__UpperCamelCase				:      Union[str, Any]									=       bytearray([self.byte_decoder[c] for c in text]      ).decode("utf-8"			,  errors=self.errors      )
							return text
				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	Tuple[str]:
							if not os.path.isdir(_UpperCAmelCase      ):
										logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
										return
							__UpperCamelCase				:      Union[str, Any]									=       os.path.join(
							    _UpperCAmelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
							__UpperCamelCase				:      Any									=       os.path.join(
							    _UpperCAmelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

							with open(_UpperCAmelCase			,  "w"			,  encoding="utf-8"      ) as f:
										f.write(json.dumps(self.encoder			,  indent=2			,  sort_keys=_UpperCAmelCase			,  ensure_ascii=_UpperCAmelCase      ) + "\n"      )

							__UpperCamelCase				:      List[str]									=       0
							with open(_UpperCAmelCase			,  "w"			,  encoding="utf-8"      ) as writer:
										writer.write("#version: 0.2\n"      )
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,  key=lambda _UpperCAmelCase      : kv[1]      ):
													if index != token_index:
																logger.warning(
																    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																    " Please check that the tokenizer is not corrupted!"      )
																__UpperCamelCase				:      int									=       token_index
													writer.write(" ".join(_UpperCAmelCase      ) + "\n"      )
													index += 1

							return vocab_file, merge_file
				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	List[int]:
							if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							__UpperCamelCase				:      Optional[int]									=       [self.cls_token_id]
							__UpperCamelCase				:      Union[str, Any]									=       [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep
				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None			,  _UpperCAmelCase = False      )     ->	List[int]:
							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=_UpperCAmelCase			,  token_ids_a=_UpperCAmelCase			,  already_has_special_tokens=_UpperCAmelCase      )

							if token_ids_a is None:
										return [1] + ([0] * len(_UpperCAmelCase      )) + [1]
							return [1] + ([0] * len(_UpperCAmelCase      )) + [1, 1] + ([0] * len(_UpperCAmelCase      )) + [1]
				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	List[int]:
							__UpperCamelCase				:      Optional[int]									=       [self.sep_token_id]
							__UpperCamelCase				:      List[Any]									=       [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep      ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase=False			,  **_UpperCAmelCase      )     ->	Optional[Any]:
							__UpperCamelCase				:      Union[str, Any]									=       kwargs.pop("add_prefix_space"			,  self.add_prefix_space      )
							if (is_split_into_words or add_prefix_space) and (len(_UpperCAmelCase      ) > 0 and not text[0].isspace()):
										__UpperCamelCase				:      Dict									=       " " + text
							return (text, kwargs)









'''simple docstring'''






from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


_lowerCAmelCase				     = logging.get_logger(__name__)

_lowerCAmelCase				     = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCAmelCase				     = {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
    '''tokenizer_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/tokenizer.json'''},
}

_lowerCAmelCase				     = {
    '''openai-gpt''': 512,
}



class  A		(			SCREAMING_SNAKE_CASE__		):




				'''simple docstring'''





				A				     =    VOCAB_FILES_NAMES
				A				     =    PRETRAINED_VOCAB_FILES_MAP
				A				     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A				     =    ["input_ids", "attention_mask"]
				A				     =    OpenAIGPTTokenizer
				def __init__(self			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase="<unk>"			,  **_UpperCAmelCase      )     ->	str:
							super().__init__(_UpperCAmelCase			,  _UpperCAmelCase			,  tokenizer_file=_UpperCAmelCase			,  unk_token=_UpperCAmelCase			,  **_UpperCAmelCase      )
				@property
				def 	a_	(self      )     ->	Dict:
							return True





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	Tuple[str]:
							__UpperCamelCase				:      Optional[Any]									=       self._tokenizer.model.save(_UpperCAmelCase			,  name=_UpperCAmelCase      )
							return tuple(_UpperCAmelCase      )



