
'''simple docstring'''



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__A       =      logging.get_logger(__name__)

__A       =      {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__A       =      {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

__A       =      {
    "openai-gpt": 512,
}





def 	_A						(							lowercase__		):
						lowercase__								=    text.strip()
						if not text:
												return []
						lowercase__								=    text.split()
						return tokens





class      A							(    __UpperCAmelCase			):



			def __init__(   self    ,       lowerCamelCase__=True    ,       lowerCamelCase__=None    ,       lowerCamelCase__=True    ,       lowerCamelCase__=None    ,       lowerCamelCase__=True    ,       )     ->						Optional[int]:




									'''simple docstring'''



									if never_split is None:
															lowercase__								=    []
									lowercase__								=    do_lower_case
									lowercase__								=    set(lowerCamelCase__    )
									lowercase__								=    tokenize_chinese_chars
									lowercase__								=    strip_accents
									lowercase__								=    do_split_on_punc



			def A__	(   self    ,       lowerCamelCase__    ,       lowerCamelCase__=None    )     ->						Dict:




									'''simple docstring'''



									lowercase__								=    self.never_split.union(set(lowerCamelCase__    )    ) if never_split else self.never_split
									lowercase__								=    self._clean_text(lowerCamelCase__    )

									# This was added on November 1st, 2018 for the multilingual and Chinese
									# models. This is also applied to the English models now, but it doesn't
									# matter since the English models were not trained on any Chinese data
									# and generally don't have any Chinese data in them (there are Chinese
									# characters in the vocabulary because Wikipedia does have some Chinese
									# words in the English Wikipedia.).
									if self.tokenize_chinese_chars:
															lowercase__								=    self._tokenize_chinese_chars(lowerCamelCase__    )
									# prevents treating the same character with different unicode codepoints as different characters
									lowercase__								=    unicodedata.normalize("""NFC"""    ,       lowerCamelCase__    )
									lowercase__								=    whitespace_tokenize(lowerCamelCase__    )
									lowercase__								=    []
									for token in orig_tokens:
															if token not in never_split:
																					if self.do_lower_case:
																											lowercase__								=    token.lower()
																											if self.strip_accents is not False:
																																	lowercase__								=    self._run_strip_accents(lowerCamelCase__    )
																					elif self.strip_accents:
																											lowercase__								=    self._run_strip_accents(lowerCamelCase__    )
															split_tokens.extend(self._run_split_on_punc(lowerCamelCase__    ,       lowerCamelCase__    )    )

									lowercase__								=    whitespace_tokenize(""" """.join(lowerCamelCase__    )    )
									return output_tokens



			def A__	(   self    ,       lowerCamelCase__    )     ->						Dict:




									'''simple docstring'''



									lowercase__								=    unicodedata.normalize("""NFD"""    ,       lowerCamelCase__    )
									lowercase__								=    []
									for char in text:
															lowercase__								=    unicodedata.category(lowerCamelCase__    )
															if cat == "Mn":
																					continue
															output.append(lowerCamelCase__    )
									return "".join(lowerCamelCase__    )



			def A__	(   self    ,       lowerCamelCase__    ,       lowerCamelCase__=None    )     ->						List[str]:




									'''simple docstring'''



									if not self.do_split_on_punc or (never_split is not None and text in never_split):
															return [text]
									lowercase__								=    list(lowerCamelCase__    )
									lowercase__								=    0
									lowercase__								=    True
									lowercase__								=    []
									while i < len(lowerCamelCase__    ):
															lowercase__								=    chars[i]
															if _is_punctuation(lowerCamelCase__    ):
																					output.append([char]    )
																					lowercase__								=    True
															else:
																					if start_new_word:
																											output.append([]    )
																					lowercase__								=    False
																					output[-1].append(lowerCamelCase__    )
															i += 1

									return ["".join(lowerCamelCase__    ) for x in output]



			def A__	(   self    ,       lowerCamelCase__    )     ->						Optional[int]:




									'''simple docstring'''



									lowercase__								=    []
									for char in text:
															lowercase__								=    ord(lowerCamelCase__    )
															if self._is_chinese_char(lowerCamelCase__    ):
																					output.append(""" """    )
																					output.append(lowerCamelCase__    )
																					output.append(""" """    )
															else:
																					output.append(lowerCamelCase__    )
									return "".join(lowerCamelCase__    )



			def A__	(   self    ,       lowerCamelCase__    )     ->						Tuple:




									'''simple docstring'''



									if (
									    (cp >= 0x4e00 and cp <= 0x9fff)
									    or (cp >= 0x3400 and cp <= 0x4dbf)  #
									    or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
									    or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
									    or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
									    or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
									    or (cp >= 0xf900 and cp <= 0xfaff)
									    or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
									):  #
															return True

									return False



			def A__	(   self    ,       lowerCamelCase__    )     ->						Any:




									'''simple docstring'''



									lowercase__								=    []
									for char in text:
															lowercase__								=    ord(lowerCamelCase__    )
															if cp == 0 or cp == 0xfffd or _is_control(lowerCamelCase__    ):
																					continue
															if _is_whitespace(lowerCamelCase__    ):
																					output.append(""" """    )
															else:
																					output.append(lowerCamelCase__    )
									return "".join(lowerCamelCase__    )





def 	_A						(							lowercase__		):
						lowercase__								=    set()
						lowercase__								=    word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)		)
												lowercase__								=    char
						return pairs





def 	_A						(							lowercase__		):
						lowercase__								=    text.replace("""—"""						,						"""-"""		)
						lowercase__								=    text.replace("""–"""						,						"""-"""		)
						lowercase__								=    text.replace("""―"""						,						"""-"""		)
						lowercase__								=    text.replace("""…"""						,						"""..."""		)
						lowercase__								=    text.replace("""´"""						,						"""'"""		)
						lowercase__								=    re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""						,						r""" \1 """						,						lowercase__		)
						lowercase__								=    re.sub(r"""\s*\n\s*"""						,						""" \n """						,						lowercase__		)
						lowercase__								=    re.sub(r"""[^\S\n]+"""						,						""" """						,						lowercase__		)
						return text.strip()







class      A							(    __UpperCAmelCase			):
			lowerCamelCase							:							Tuple    						=					VOCAB_FILES_NAMES
			lowerCamelCase							:							Tuple    						=					PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase							:							str    						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase							:							Any    						=					["""input_ids""", """attention_mask"""]



			def __init__(   self    ,       lowerCamelCase__    ,       lowerCamelCase__    ,       lowerCamelCase__="<unk>"    ,       **lowerCamelCase__    )     ->						Tuple:




									'''simple docstring'''



									super().__init__(unk_token=lowerCamelCase__    ,       **lowerCamelCase__    )

									try:
															import ftfy
															from spacy.lang.en import English

															lowercase__								=    English()
															lowercase__								=    _nlp.tokenizer
															lowercase__								=    ftfy.fix_text
									except ImportError:
															logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""    )
															lowercase__								=    BasicTokenizer(do_lower_case=lowerCamelCase__    )
															lowercase__								=    None

									with open(lowerCamelCase__    ,       encoding="""utf-8"""    ) as vocab_handle:
															lowercase__								=    json.load(lowerCamelCase__    )
									lowercase__								=    {v: k for k, v in self.encoder.items()}
									with open(lowerCamelCase__    ,       encoding="""utf-8"""    ) as merges_handle:
															lowercase__								=    merges_handle.read().split("""\n"""    )[1:-1]
									lowercase__								=    [tuple(merge.split()    ) for merge in merges]
									lowercase__								=    dict(zip(lowerCamelCase__    ,       range(len(lowerCamelCase__    )    )    )    )
									lowercase__								=    {}



			@property
			def A__	(   self    )     ->						int:




									'''simple docstring'''



									return True



			@property
			def A__	(   self    )     ->						Dict:




									'''simple docstring'''



									return len(self.encoder    )



			def A__	(   self    )     ->						Optional[int]:




									'''simple docstring'''



									return dict(self.encoder    ,       **self.added_tokens_encoder    )



			def A__	(   self    ,       lowerCamelCase__    )     ->						str:




									'''simple docstring'''



									lowercase__								=    tuple(token[:-1]    ) + (token[-1] + """</w>""",)
									if token in self.cache:
															return self.cache[token]
									lowercase__								=    get_pairs(lowerCamelCase__    )

									if not pairs:
															return token + "</w>"

									while True:
															lowercase__								=    min(lowerCamelCase__    ,       key=lambda lowerCamelCase__    : self.bpe_ranks.get(lowerCamelCase__    ,       float("""inf"""    )    )    )
															if bigram not in self.bpe_ranks:
																					break
															lowercase__	,  lowercase__								=    bigram
															lowercase__								=    []
															lowercase__								=    0
															while i < len(lowerCamelCase__    ):
																					try:
																											lowercase__								=    word.index(lowerCamelCase__    ,       lowerCamelCase__    )
																					except ValueError:
																											new_word.extend(word[i:]    )
																											break
																					else:
																											new_word.extend(word[i:j]    )
																											lowercase__								=    j

																					if word[i] == first and i < len(lowerCamelCase__    ) - 1 and word[i + 1] == second:
																											new_word.append(first + second    )
																											i += 2
																					else:
																											new_word.append(word[i]    )
																											i += 1
															lowercase__								=    tuple(lowerCamelCase__    )
															lowercase__								=    new_word
															if len(lowerCamelCase__    ) == 1:
																					break
															else:
																					lowercase__								=    get_pairs(lowerCamelCase__    )
									lowercase__								=    """ """.join(lowerCamelCase__    )
									if word == "\n  </w>":
															lowercase__								=    """\n</w>"""
									lowercase__								=    word
									return word



			def A__	(   self    ,       lowerCamelCase__    )     ->						str:




									'''simple docstring'''



									lowercase__								=    []
									if self.fix_text is None:
															# Using BERT's BasicTokenizer
															lowercase__								=    self.nlp.tokenize(lowerCamelCase__    )
															for token in text:
																					split_tokens.extend(list(self.bpe(lowerCamelCase__    ).split(""" """    )    )    )
									else:
															# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
															lowercase__								=    self.nlp(text_standardize(self.fix_text(lowerCamelCase__    )    )    )
															for token in text:
																					split_tokens.extend(list(self.bpe(token.text.lower()    ).split(""" """    )    )    )
									return split_tokens



			def A__	(   self    ,       lowerCamelCase__    )     ->						Dict:




									'''simple docstring'''



									return self.encoder.get(lowerCamelCase__    ,       self.encoder.get(self.unk_token    )    )



			def A__	(   self    ,       lowerCamelCase__    )     ->						Optional[Any]:




									'''simple docstring'''



									return self.decoder.get(lowerCamelCase__    ,       self.unk_token    )



			def A__	(   self    ,       lowerCamelCase__    )     ->						List[Any]:




									'''simple docstring'''



									lowercase__								=    """""".join(lowerCamelCase__    ).replace("""</w>"""    ,       """ """    ).strip()
									return out_string



			def A__	(   self    ,       lowerCamelCase__    ,       lowerCamelCase__ = None    )     ->						Tuple[str]:




									'''simple docstring'''



									if not os.path.isdir(lowerCamelCase__    ):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
															return
									lowercase__								=    os.path.join(
									    lowerCamelCase__    ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
									lowercase__								=    os.path.join(
									    lowerCamelCase__    ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]    )

									with open(lowerCamelCase__    ,       """w"""    ,       encoding="""utf-8"""    ) as f:
															f.write(json.dumps(self.encoder    ,       indent=2    ,       sort_keys=lowerCamelCase__    ,       ensure_ascii=lowerCamelCase__    ) + """\n"""    )

									lowercase__								=    0
									with open(lowerCamelCase__    ,       """w"""    ,       encoding="""utf-8"""    ) as writer:
															writer.write("""#version: 0.2\n"""    )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,       key=lambda lowerCamelCase__    : kv[1]    ):
																					if index != token_index:
																											logger.warning(
																											    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																											    """ Please check that the tokenizer is not corrupted!"""    )
																											lowercase__								=    token_index
																					writer.write(""" """.join(lowerCamelCase__    ) + """\n"""    )
																					index += 1

									return vocab_file, merge_file








'''simple docstring'''



import warnings

from .generation import FlaxGenerationMixin





class      A							(    __UpperCAmelCase			):
			# warning at import time
			warnings.warn(
			    """Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will """
			    """be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead."""						,  __UpperCAmelCase						,  )






