



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase								=   logging.get_logger(__name__)

UpperCAmelCase								=   {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

UpperCAmelCase								=   {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

UpperCAmelCase								=   {
    "openai-gpt": 512,
}

def      A_ (		__a       :			List[Any]						):

				"""simple docstring"""
				a__			 =						text.strip()
				if not text:
								return []
				a__			 =						text.split()
				return tokens

class 	__snake_case (				UpperCAmelCase__):







							'''simple docstring'''






							def __init__(							self					,					a_=True					,					a_=None					,					a_=True					,					a_=None					,					a_=True					,					):






											if never_split is None:
															a__			 =						[]
											a__			 =						do_lower_case
											a__			 =						set(__lowercase						)
											a__			 =						tokenize_chinese_chars
											a__			 =						strip_accents
											a__			 =						do_split_on_punc


							def _a (							self					,					a_					,					a_=None						):
											a__			 =						self.never_split.union(set(__lowercase						)						) if never_split else self.never_split
											a__			 =						self._clean_text(__lowercase						)

											# This was added on November 1st, 2018 for the multilingual and Chinese
											# models. This is also applied to the English models now, but it doesn't
											# matter since the English models were not trained on any Chinese data
											# and generally don't have any Chinese data in them (there are Chinese
											# characters in the vocabulary because Wikipedia does have some Chinese
											# words in the English Wikipedia.).
											if self.tokenize_chinese_chars:
															a__			 =						self._tokenize_chinese_chars(__lowercase						)
											# prevents treating the same character with different unicode codepoints as different characters
											a__			 =						unicodedata.normalize("""NFC"""					,					__lowercase						)
											a__			 =						whitespace_tokenize(__lowercase						)
											a__			 =						[]
											for token in orig_tokens:
															if token not in never_split:
																			if self.do_lower_case:
																							a__			 =						token.lower()
																							if self.strip_accents is not False:
																											a__			 =						self._run_strip_accents(__lowercase						)
																			elif self.strip_accents:
																							a__			 =						self._run_strip_accents(__lowercase						)
															split_tokens.extend(self._run_split_on_punc(__lowercase					,					__lowercase						)						)

											a__			 =						whitespace_tokenize(""" """.join(__lowercase						)						)
											return output_tokens


							def _a (							self					,					a_						):
											a__			 =						unicodedata.normalize("""NFD"""					,					__lowercase						)
											a__			 =						[]
											for char in text:
															a__			 =						unicodedata.category(__lowercase						)
															if cat == "Mn":
																			continue
															output.append(__lowercase						)
											return "".join(__lowercase						)


							def _a (							self					,					a_					,					a_=None						):






											if not self.do_split_on_punc or (never_split is not None and text in never_split):
															return [text]
											a__			 =						list(__lowercase						)
											a__			 =						0
											a__			 =						True
											a__			 =						[]
											while i < len(__lowercase						):
															a__			 =						chars[i]
															if _is_punctuation(__lowercase						):
																			output.append([char]						)
																			a__			 =						True
															else:
																			if start_new_word:
																							output.append([]						)
																			a__			 =						False
																			output[-1].append(__lowercase						)
															i += 1

											return ["".join(__lowercase						) for x in output]


							def _a (							self					,					a_						):
											a__			 =						[]
											for char in text:
															a__			 =						ord(__lowercase						)
															if self._is_chinese_char(__lowercase						):
																			output.append(""" """						)
																			output.append(__lowercase						)
																			output.append(""" """						)
															else:
																			output.append(__lowercase						)
											return "".join(__lowercase						)


							def _a (							self					,					a_						):






											if (
											    (cp >= 0x4e00 and cp <= 0x9fff)
											    or (cp >= 0x3400 and cp <= 0x4dbf)  #
											    or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
											    or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
											    or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
											    or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
											    or (cp >= 0xf900 and cp <= 0xfaff)
											    or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
											):  #
															return True

											return False







							def _a (							self					,					a_						):
											a__			 =						[]
											for char in text:
															a__			 =						ord(__lowercase						)
															if cp == 0 or cp == 0xfffd or _is_control(__lowercase						):
																			continue
															if _is_whitespace(__lowercase						):
																			output.append(""" """						)
															else:
																			output.append(__lowercase						)
											return "".join(__lowercase						)




def      A_ (		__a       :			int						):

				"""simple docstring"""
				a__			 =						set()
				a__			 =						word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)						)
								a__			 =						char
				return pairs

def      A_ (		__a       :			int						):

				"""simple docstring"""
				a__			 =						text.replace("""—"""	,						"""-"""						)
				a__			 =						text.replace("""–"""	,						"""-"""						)
				a__			 =						text.replace("""―"""	,						"""-"""						)
				a__			 =						text.replace("""…"""	,						"""..."""						)
				a__			 =						text.replace("""´"""	,						"""'"""						)
				a__			 =						re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""	,						r""" \1 """	,						_A						)
				a__			 =						re.sub(r"""\s*\n\s*"""	,						""" \n """	,						_A						)
				a__			 =						re.sub(r"""[^\S\n]+"""	,						""" """	,						_A						)
				return text.strip()




class 	__snake_case (				UpperCAmelCase__):







							'''simple docstring'''






							UpperCamelCase__			: Optional[int]             =   VOCAB_FILES_NAMES
							UpperCamelCase__			: int             =   PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__			: int             =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__			: Dict             =   ["""input_ids""", """attention_mask"""]


							def __init__(							self					,					a_					,					a_					,					a_="<unk>"					,					**a_						):
											super().__init__(unk_token=__lowercase					,					**__lowercase						)

											try:
															import ftfy
															from spacy.lang.en import English

															a__			 =						English()
															a__			 =						_nlp.tokenizer
															a__			 =						ftfy.fix_text
											except ImportError:
															logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""						)
															a__			 =						BasicTokenizer(do_lower_case=__lowercase						)
															a__			 =						None

											with open(__lowercase					,					encoding="""utf-8"""						) as vocab_handle:
															a__			 =						json.load(__lowercase						)
											a__			 =						{v: k for k, v in self.encoder.items()}
											with open(__lowercase					,					encoding="""utf-8"""						) as merges_handle:
															a__			 =						merges_handle.read().split("""\n"""						)[1:-1]
											a__			 =						[tuple(merge.split()						) for merge in merges]
											a__			 =						dict(zip(__lowercase					,					range(len(__lowercase						)						)						)						)
											a__			 =						{}


							@property
							def _a (							self						):
											return True


							@property
							def _a (							self						):
											return len(self.encoder						)


							def _a (							self						):
											return dict(self.encoder					,					**self.added_tokens_encoder						)


							def _a (							self					,					a_						):
											a__			 =						tuple(token[:-1]						) + (token[-1] + """</w>""",)
											if token in self.cache:
															return self.cache[token]
											a__			 =						get_pairs(__lowercase						)

											if not pairs:
															return token + "</w>"

											while True:
															a__			 =						min(__lowercase					,					key=lambda a_						: self.bpe_ranks.get(__lowercase					,					float("""inf"""						)						)						)
															if bigram not in self.bpe_ranks:
																			break
															a__ ,     a__			 =						bigram
															a__			 =						[]
															a__			 =						0
															while i < len(__lowercase						):
																			try:
																							a__			 =						word.index(__lowercase					,					__lowercase						)
																			except ValueError:
																							new_word.extend(word[i:]						)
																							break
																			else:
																							new_word.extend(word[i:j]						)
																							a__			 =						j

																			if word[i] == first and i < len(__lowercase						) - 1 and word[i + 1] == second:
																							new_word.append(first + second						)
																							i += 2
																			else:
																							new_word.append(word[i]						)
																							i += 1
															a__			 =						tuple(__lowercase						)
															a__			 =						new_word
															if len(__lowercase						) == 1:
																			break
															else:
																			a__			 =						get_pairs(__lowercase						)
											a__			 =						""" """.join(__lowercase						)
											if word == "\n  </w>":
															a__			 =						"""\n</w>"""
											a__			 =						word
											return word


							def _a (							self					,					a_						):
											a__			 =						[]
											if self.fix_text is None:
															# Using BERT's BasicTokenizer
															a__			 =						self.nlp.tokenize(__lowercase						)
															for token in text:
																			split_tokens.extend(list(self.bpe(__lowercase						).split(""" """						)						)						)
											else:
															# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
															a__			 =						self.nlp(text_standardize(self.fix_text(__lowercase						)						)						)
															for token in text:
																			split_tokens.extend(list(self.bpe(token.text.lower()						).split(""" """						)						)						)
											return split_tokens


							def _a (							self					,					a_						):
											return self.encoder.get(__lowercase					,					self.encoder.get(self.unk_token						)						)


							def _a (							self					,					a_						):
											return self.decoder.get(__lowercase					,					self.unk_token						)


							def _a (							self					,					a_						):
											a__			 =						"""""".join(__lowercase						).replace("""</w>"""					,					""" """						).strip()
											return out_string







							def _a (							self					,					a_					,					a_ = None						):






											if not os.path.isdir(__lowercase						):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
															return
											a__			 =						os.path.join(
											    __lowercase					,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
											a__			 =						os.path.join(
											    __lowercase					,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

											with open(__lowercase					,					"""w"""					,					encoding="""utf-8"""						) as f:
															f.write(json.dumps(self.encoder					,					indent=2					,					sort_keys=__lowercase					,					ensure_ascii=__lowercase						) + """\n"""						)

											a__			 =						0
											with open(__lowercase					,					"""w"""					,					encoding="""utf-8"""						) as writer:
															writer.write("""#version: 0.2\n"""						)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,					key=lambda a_						: kv[1]						):
																			if index != token_index:
																							logger.warning(
																							    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																							    """ Please check that the tokenizer is not corrupted!"""						)
																							a__			 =						token_index
																			writer.write(""" """.join(__lowercase						) + """\n"""						)
																			index += 1

											return vocab_file, merge_file






from __future__ import annotations

def      A_ (		__a       :			list[int | float]						):

				"""simple docstring"""
				if len(__a						) == 0:
								raise ValueError("""find_max() arg is an empty sequence"""						)
				a__			 =						nums[0]
				for x in nums:
								if x > max_num:
												a__			 =						x
				return max_num


if __name__ == "__main__":
	import doctest

	doctest.testmod(verbose=True)

