




"""simple docstring"""




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a_            =   logging.get_logger(__name__)

a_            =   {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a_            =   {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

a_            =   {
    'openai-gpt': 5_1_2,
}







def       __UpperCAmelCase      (				__UpperCamelCase					):
			__lowercase	:       Dict				=   text.strip()
			if not text:
						return []
			__lowercase	:       Optional[Any]				=   text.split()
			return tokens

class        UpperCAmelCase_  (	snake_case						):
		def __init__(		self							,		UpperCamelCase_=True							,		UpperCamelCase_=None							,		UpperCamelCase_=True							,		UpperCamelCase_=None							,		UpperCamelCase_=True							,		)		->   Tuple:
					if never_split is None:
								__lowercase	:       Optional[int]				=   []
					__lowercase	:       Dict				=   do_lower_case
					__lowercase	:       Optional[Any]				=   set(UpperCamelCase_   )
					__lowercase	:       Optional[Any]				=   tokenize_chinese_chars
					__lowercase	:       Optional[Any]				=   strip_accents
					__lowercase	:       Any				=   do_split_on_punc
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_							,		UpperCamelCase_=None   )		->   Any:
					__lowercase	:       str				=   self.never_split.union(set(UpperCamelCase_   )   ) if never_split else self.never_split
					__lowercase	:       Any				=   self._clean_text(UpperCamelCase_   )

					# This was added on November 1st, 2018 for the multilingual and Chinese
					# models. This is also applied to the English models now, but it doesn't
					# matter since the English models were not trained on any Chinese data
					# and generally don't have any Chinese data in them (there are Chinese
					# characters in the vocabulary because Wikipedia does have some Chinese
					# words in the English Wikipedia.).
					if self.tokenize_chinese_chars:
								__lowercase	:       Any				=   self._tokenize_chinese_chars(UpperCamelCase_   )
					# prevents treating the same character with different unicode codepoints as different characters
					__lowercase	:       Tuple				=   unicodedata.normalize('''NFC'''							,		UpperCamelCase_   )
					__lowercase	:       List[Any]				=   whitespace_tokenize(UpperCamelCase_   )
					__lowercase	:       Optional[int]				=   []
					for token in orig_tokens:
								if token not in never_split:
											if self.do_lower_case:
														__lowercase	:       List[str]				=   token.lower()
														if self.strip_accents is not False:
																	__lowercase	:       int				=   self._run_strip_accents(UpperCamelCase_   )
											elif self.strip_accents:
														__lowercase	:       Dict				=   self._run_strip_accents(UpperCamelCase_   )
								split_tokens.extend(self._run_split_on_punc(UpperCamelCase_							,		UpperCamelCase_   )   )

					__lowercase	:       Optional[Any]				=   whitespace_tokenize(''' '''.join(UpperCamelCase_   )   )
					return output_tokens
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   Union[str, Any]:
					__lowercase	:       Optional[Any]				=   unicodedata.normalize('''NFD'''							,		UpperCamelCase_   )
					__lowercase	:       Optional[int]				=   []
					for char in text:
								__lowercase	:       Tuple				=   unicodedata.category(UpperCamelCase_   )
								if cat == "Mn":
											continue
								output.append(UpperCamelCase_   )
					return "".join(UpperCamelCase_   )
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_							,		UpperCamelCase_=None   )		->   Tuple:
					if not self.do_split_on_punc or (never_split is not None and text in never_split):
								return [text]
					__lowercase	:       int				=   list(UpperCamelCase_   )
					__lowercase	:       Tuple				=   0
					__lowercase	:       Tuple				=   True
					__lowercase	:       List[Any]				=   []
					while i < len(UpperCamelCase_   ):
								__lowercase	:       Tuple				=   chars[i]
								if _is_punctuation(UpperCamelCase_   ):
											output.append([char]   )
											__lowercase	:       List[Any]				=   True
								else:
											if start_new_word:
														output.append([]   )
											__lowercase	:       Dict				=   False
											output[-1].append(UpperCamelCase_   )
								i += 1

					return ["".join(UpperCamelCase_   ) for x in output]
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   str:
					__lowercase	:       List[str]				=   []
					for char in text:
								__lowercase	:       Any				=   ord(UpperCamelCase_   )
								if self._is_chinese_char(UpperCamelCase_   ):
											output.append(''' '''   )
											output.append(UpperCamelCase_   )
											output.append(''' '''   )
								else:
											output.append(UpperCamelCase_   )
					return "".join(UpperCamelCase_   )
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   List[str]:
					# This defines a "chinese character" as anything in the CJK Unicode block:
					#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
					#
					# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
					# despite its name. The modern Korean Hangul alphabet is a different block,
					# as is Japanese Hiragana and Katakana. Those alphabets are used to write
					# space-separated words, so they are not treated specially and handled
					# like the all of the other languages.
					if (
					    (cp >= 0x4_e_0_0 and cp <= 0x9_f_f_f)
					    or (cp >= 0x3_4_0_0 and cp <= 0x4_d_b_f)  #
					    or (cp >= 0x2_0_0_0_0 and cp <= 0x2_a_6_d_f)  #
					    or (cp >= 0x2_a_7_0_0 and cp <= 0x2_b_7_3_f)  #
					    or (cp >= 0x2_b_7_4_0 and cp <= 0x2_b_8_1_f)  #
					    or (cp >= 0x2_b_8_2_0 and cp <= 0x2_c_e_a_f)  #
					    or (cp >= 0xf_9_0_0 and cp <= 0xf_a_f_f)
					    or (cp >= 0x2_f_8_0_0 and cp <= 0x2_f_a_1_f)  #
					):  #
								return True

					return False
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   Dict:
					__lowercase	:       Any				=   []
					for char in text:
								__lowercase	:       str				=   ord(UpperCamelCase_   )
								if cp == 0 or cp == 0xf_f_f_d or _is_control(UpperCamelCase_   ):
											continue
								if _is_whitespace(UpperCamelCase_   ):
											output.append(''' '''   )
								else:
											output.append(UpperCamelCase_   )
					return "".join(UpperCamelCase_   )







def       __UpperCAmelCase      (				__UpperCamelCase					):
			__lowercase	:       Dict				=   set()
			__lowercase	:       Any				=   word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)					)
						__lowercase	:       List[str]				=   char
			return pairs







def       __UpperCAmelCase      (				__UpperCamelCase					):
			__lowercase	:       Any				=   text.replace('''—'''					,							'''-'''					)
			__lowercase	:       Tuple				=   text.replace('''–'''					,							'''-'''					)
			__lowercase	:       int				=   text.replace('''―'''					,							'''-'''					)
			__lowercase	:       int				=   text.replace('''…'''					,							'''...'''					)
			__lowercase	:       Dict				=   text.replace('''´'''					,							'''\''''					)
			__lowercase	:       Optional[int]				=   re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''					,							R''' \1 '''					,							__UpperCamelCase					)
			__lowercase	:       Optional[int]				=   re.sub(R'''\s*\n\s*'''					,							''' \n '''					,							__UpperCamelCase					)
			__lowercase	:       Optional[int]				=   re.sub(R'''[^\S\n]+'''					,							''' '''					,							__UpperCamelCase					)
			return text.strip()


class        UpperCAmelCase_  (	snake_case						):
		UpperCamelCase									=VOCAB_FILES_NAMES
		UpperCamelCase									=PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase									=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCamelCase									=["input_ids", "attention_mask"]
		def __init__(		self							,		UpperCamelCase_							,		UpperCamelCase_							,		UpperCamelCase_="<unk>"							,		**UpperCamelCase_   )		->   List[Any]:
					super().__init__(unk_token=UpperCamelCase_							,		**UpperCamelCase_   )

					try:
								import ftfy
								from spacy.lang.en import English

								__lowercase	:       Union[str, Any]				=   English()
								__lowercase	:       Any				=   _nlp.tokenizer
								__lowercase	:       Tuple				=   ftfy.fix_text
					except ImportError:
								logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''   )
								__lowercase	:       Union[str, Any]				=   BasicTokenizer(do_lower_case=UpperCamelCase_   )
								__lowercase	:       int				=   None

					with open(UpperCamelCase_							,		encoding='''utf-8'''   ) as vocab_handle:
								__lowercase	:       Optional[int]				=   json.load(UpperCamelCase_   )
					__lowercase	:       str				=   {v: k for k, v in self.encoder.items()}
					with open(UpperCamelCase_							,		encoding='''utf-8'''   ) as merges_handle:
								__lowercase	:       Any				=   merges_handle.read().split('''\n'''   )[1:-1]
					__lowercase	:       Optional[int]				=   [tuple(merge.split()   ) for merge in merges]
					__lowercase	:       List[Any]				=   dict(zip(UpperCamelCase_							,		range(len(UpperCamelCase_   )   )   )   )
					__lowercase	:       List[Any]				=   {}
		@property
		def 					_lowerCamelCase    (		self   )		->   Optional[int]:
					return True
		@property
		def 					_lowerCamelCase    (		self   )		->   Any:
					return len(self.encoder   )
		def 					_lowerCamelCase    (		self   )		->   Optional[int]:
					return dict(self.encoder							,		**self.added_tokens_encoder   )
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   List[str]:
					__lowercase	:       Optional[Any]				=   tuple(token[:-1]   ) + (token[-1] + '''</w>''',)
					if token in self.cache:
								return self.cache[token]
					__lowercase	:       int				=   get_pairs(UpperCamelCase_   )

					if not pairs:
								return token + "</w>"

					while True:
								__lowercase	:       str				=   min(UpperCamelCase_							,		key=lambda UpperCamelCase_   : self.bpe_ranks.get(UpperCamelCase_							,		float('''inf'''   )   )   )
								if bigram not in self.bpe_ranks:
											break
								__lowercase	:       List[str]				=   bigram
								__lowercase	:       int				=   []
								__lowercase	:       str				=   0
								while i < len(UpperCamelCase_   ):
											try:
														__lowercase	:       int				=   word.index(UpperCamelCase_							,		UpperCamelCase_   )
											except ValueError:
														new_word.extend(word[i:]   )
														break
											else:
														new_word.extend(word[i:j]   )
														__lowercase	:       Dict				=   j

											if word[i] == first and i < len(UpperCamelCase_   ) - 1 and word[i + 1] == second:
														new_word.append(first + second   )
														i += 2
											else:
														new_word.append(word[i]   )
														i += 1
								__lowercase	:       List[Any]				=   tuple(UpperCamelCase_   )
								__lowercase	:       List[str]				=   new_word
								if len(UpperCamelCase_   ) == 1:
											break
								else:
											__lowercase	:       Any				=   get_pairs(UpperCamelCase_   )
					__lowercase	:       List[Any]				=   ''' '''.join(UpperCamelCase_   )
					if word == "\n  </w>":
								__lowercase	:       Optional[Any]				=   '''\n</w>'''
					__lowercase	:       Optional[Any]				=   word
					return word
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   Any:
					__lowercase	:       List[Any]				=   []
					if self.fix_text is None:
								# Using BERT's BasicTokenizer
								__lowercase	:       Any				=   self.nlp.tokenize(UpperCamelCase_   )
								for token in text:
											split_tokens.extend(list(self.bpe(UpperCamelCase_   ).split(''' '''   )   )   )
					else:
								# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
								__lowercase	:       Tuple				=   self.nlp(text_standardize(self.fix_text(UpperCamelCase_   )   )   )
								for token in text:
											split_tokens.extend(list(self.bpe(token.text.lower()   ).split(''' '''   )   )   )
					return split_tokens
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   Union[str, Any]:
					return self.encoder.get(UpperCamelCase_							,		self.encoder.get(self.unk_token   )   )
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   List[str]:
					return self.decoder.get(UpperCamelCase_							,		self.unk_token   )
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_   )		->   List[Any]:
					__lowercase	:       Optional[Any]				=   ''''''.join(UpperCamelCase_   ).replace('''</w>'''							,		''' '''   ).strip()
					return out_string
		def 					_lowerCamelCase    (		self							,		UpperCamelCase_							,		UpperCamelCase_ = None   )		->   Tuple[str]:
					if not os.path.isdir(UpperCamelCase_   ):
								logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
								return
					__lowercase	:       Union[str, Any]				=   os.path.join(
					    UpperCamelCase_							,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
					__lowercase	:       Union[str, Any]				=   os.path.join(
					    UpperCamelCase_							,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']   )

					with open(UpperCamelCase_							,		'''w'''							,		encoding='''utf-8'''   ) as f:
								f.write(json.dumps(self.encoder							,		indent=2							,		sort_keys=UpperCamelCase_							,		ensure_ascii=UpperCamelCase_   ) + '''\n'''   )

					__lowercase	:       Union[str, Any]				=   0
					with open(UpperCamelCase_							,		'''w'''							,		encoding='''utf-8'''   ) as writer:
								writer.write('''#version: 0.2\n'''   )
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,		key=lambda UpperCamelCase_   : kv[1]   ):
											if index != token_index:
														logger.warning(
														    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
														    ''' Please check that the tokenizer is not corrupted!'''   )
														__lowercase	:       str				=   token_index
											writer.write(''' '''.join(UpperCamelCase_   ) + '''\n'''   )
											index += 1

					return vocab_file, merge_file






"""simple docstring"""




def       __UpperCAmelCase      (				__UpperCamelCase					):
			if len(__UpperCamelCase					) <= 1:
						return data
			else:
						return [
						    *quick_sort([e for e in data[1:] if e <= data[0]]					),
						    data[0],
						    *quick_sort([e for e in data[1:] if e > data[0]]					),
						]


if __name__ == "__main__":
			import doctest

			doctest.testmod()



