



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a__						=		logging.get_logger(__name__)

a__						=		{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a__						=		{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

a__						=		{
    """openai-gpt""": 5_12,
}



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		Tuple			)					->       Tuple:
 _snake_case      :					Tuple					=   text.strip()
 if not text:
  return []
 _snake_case      :					Any					=   text.split()
 return tokens







class 						snake_case  (		SCREAMING_SNAKE_CASE_					):



   '''simple docstring'''






   def __init__(   self							:     List[Any]			,						lowerCAmelCase							:     Any=True			,						lowerCAmelCase							:     List[Any]=None			,						lowerCAmelCase							:     Tuple=True			,						lowerCAmelCase							:     Any=None			,						lowerCAmelCase							:     Tuple=True			,						)			->			int:

    """simple docstring"""







    if never_split is None:
     _snake_case      :					int					=   []
    _snake_case      :					List[Any]					=   do_lower_case
    _snake_case      :					Optional[Any]					=   set(lowerCAmelCase)
    _snake_case      :					List[Any]					=   tokenize_chinese_chars
    _snake_case      :					int					=   strip_accents
    _snake_case      :					int					=   do_split_on_punc

   def        UpperCamelCase_					(   self							:     Optional[int]			,						lowerCAmelCase							:     Union[str, Any]			,						lowerCAmelCase							:     int=None)			->			Dict:

    """simple docstring"""







    _snake_case      :					Any					=   self.never_split.union(set(lowerCAmelCase)) if never_split else self.never_split
    _snake_case      :					Tuple					=   self._clean_text(lowerCAmelCase)

    # This was added on November 1st, 2018 for the multilingual and Chinese
    # models. This is also applied to the English models now, but it doesn't
    # matter since the English models were not trained on any Chinese data
    # and generally don't have any Chinese data in them (there are Chinese
    # characters in the vocabulary because Wikipedia does have some Chinese
    # words in the English Wikipedia.).
    if self.tokenize_chinese_chars:
     _snake_case      :					List[str]					=   self._tokenize_chinese_chars(lowerCAmelCase)
    # prevents treating the same character with different unicode codepoints as different characters
    _snake_case      :					Union[str, Any]					=   unicodedata.normalize("""NFC"""			,						lowerCAmelCase)
    _snake_case      :					List[Any]					=   whitespace_tokenize(lowerCAmelCase)
    _snake_case      :					List[str]					=   []
    for token in orig_tokens:
     if token not in never_split:
      if self.do_lower_case:
       _snake_case      :					Dict					=   token.lower()
       if self.strip_accents is not False:
        _snake_case      :					List[Any]					=   self._run_strip_accents(lowerCAmelCase)
      elif self.strip_accents:
       _snake_case      :					Optional[int]					=   self._run_strip_accents(lowerCAmelCase)
     split_tokens.extend(self._run_split_on_punc(lowerCAmelCase			,						lowerCAmelCase))

    _snake_case      :					Optional[Any]					=   whitespace_tokenize(""" """.join(lowerCAmelCase))
    return output_tokens

   def        UpperCamelCase_					(   self							:     str			,						lowerCAmelCase							:     int)			->			List[Any]:

    """simple docstring"""







    _snake_case      :					Optional[int]					=   unicodedata.normalize("""NFD"""			,						lowerCAmelCase)
    _snake_case      :					Dict					=   []
    for char in text:
     _snake_case      :					Tuple					=   unicodedata.category(lowerCAmelCase)
     if cat == "Mn":
      continue
     output.append(lowerCAmelCase)
    return "".join(lowerCAmelCase)

   def        UpperCamelCase_					(   self							:     str			,						lowerCAmelCase							:     Any			,						lowerCAmelCase							:     Tuple=None)			->			List[str]:

    """simple docstring"""







    if not self.do_split_on_punc or (never_split is not None and text in never_split):
     return [text]
    _snake_case      :					Any					=   list(lowerCAmelCase)
    _snake_case      :					List[Any]					=   0
    _snake_case      :					List[str]					=   True
    _snake_case      :					Any					=   []
    while i < len(lowerCAmelCase):
     _snake_case      :					Any					=   chars[i]
     if _is_punctuation(lowerCAmelCase):
      output.append([char])
      _snake_case      :					List[str]					=   True
     else:
      if start_new_word:
       output.append([])
      _snake_case      :					str					=   False
      output[-1].append(lowerCAmelCase)
     i += 1

    return ["".join(lowerCAmelCase) for x in output]

   def        UpperCamelCase_					(   self							:     List[str]			,						lowerCAmelCase							:     Optional[Any])			->			List[Any]:

    """simple docstring"""







    _snake_case      :					Optional[int]					=   []
    for char in text:
     _snake_case      :					Union[str, Any]					=   ord(lowerCAmelCase)
     if self._is_chinese_char(lowerCAmelCase):
      output.append(""" """)
      output.append(lowerCAmelCase)
      output.append(""" """)
     else:
      output.append(lowerCAmelCase)
    return "".join(lowerCAmelCase)

   def        UpperCamelCase_					(   self							:     Dict			,						lowerCAmelCase							:     List[str])			->			Optional[int]:

    """simple docstring"""







    if (
        (cp >= 0X4_e_0_0 and cp <= 0X9_f_f_f)
        or (cp >= 0X3_4_0_0 and cp <= 0X4_d_b_f)  #
        or (cp >= 0X2_0_0_0_0 and cp <= 0X2_a_6_d_f)  #
        or (cp >= 0X2_a_7_0_0 and cp <= 0X2_b_7_3_f)  #
        or (cp >= 0X2_b_7_4_0 and cp <= 0X2_b_8_1_f)  #
        or (cp >= 0X2_b_8_2_0 and cp <= 0X2_c_e_a_f)  #
        or (cp >= 0Xf_9_0_0 and cp <= 0Xf_a_f_f)
        or (cp >= 0X2_f_8_0_0 and cp <= 0X2_f_a_1_f)  #
    ):  #
     return True

    return False


   def        UpperCamelCase_					(   self							:     Tuple			,						lowerCAmelCase							:     Optional[Any])			->			str:

    """simple docstring"""







    _snake_case      :					Any					=   []
    for char in text:
     _snake_case      :					Optional[Any]					=   ord(lowerCAmelCase)
     if cp == 0 or cp == 0Xf_f_f_d or _is_control(lowerCAmelCase):
      continue
     if _is_whitespace(lowerCAmelCase):
      output.append(""" """)
     else:
      output.append(lowerCAmelCase)
    return "".join(lowerCAmelCase)



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		Optional[Any]			)					->       Dict:
 _snake_case      :					int					=   set()
 _snake_case      :					int					=   word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)			)
  _snake_case      :					Tuple					=   char
 return pairs



def  lowercase    (				SCREAMING_SNAKE_CASE__    :		Tuple			)					->       Tuple:
 _snake_case      :					Tuple					=   text.replace("""—"""			,    """-"""			)
 _snake_case      :					Dict					=   text.replace("""–"""			,    """-"""			)
 _snake_case      :					Tuple					=   text.replace("""―"""			,    """-"""			)
 _snake_case      :					str					=   text.replace("""…"""			,    """..."""			)
 _snake_case      :					Optional[int]					=   text.replace("""´"""			,    """'"""			)
 _snake_case      :					Union[str, Any]					=   re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""			,    R""" \1 """			,    SCREAMING_SNAKE_CASE__			)
 _snake_case      :					Dict					=   re.sub(R"""\s*\n\s*"""			,    """ \n """			,    SCREAMING_SNAKE_CASE__			)
 _snake_case      :					str					=   re.sub(R"""[^\S\n]+"""			,    """ """			,    SCREAMING_SNAKE_CASE__			)
 return text.strip()







class 						snake_case  (		SCREAMING_SNAKE_CASE_					):



   '''simple docstring'''






   snake_case_   :	Dict       =						VOCAB_FILES_NAMES
   snake_case_   :	str       =						PRETRAINED_VOCAB_FILES_MAP
   snake_case_   :	Any       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   snake_case_   :	Optional[int]       =						["""input_ids""", """attention_mask"""]

   def __init__(   self							:     Union[str, Any]			,						lowerCAmelCase							:     List[str]			,						lowerCAmelCase							:     List[Any]			,						lowerCAmelCase							:     Dict="<unk>"			,						**lowerCAmelCase							:     Tuple)			->			str:

    """simple docstring"""







    super().__init__(unk_token=lowerCAmelCase			,						**lowerCAmelCase)

    try:
     import ftfy
     from spacy.lang.en import English

     _snake_case      :					Optional[int]					=   English()
     _snake_case      :					int					=   _nlp.tokenizer
     _snake_case      :					Optional[int]					=   ftfy.fix_text
    except ImportError:
     logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.""")
     _snake_case      :					Union[str, Any]					=   BasicTokenizer(do_lower_case=lowerCAmelCase)
     _snake_case      :					Dict					=   None

    with open(lowerCAmelCase			,						encoding="""utf-8""") as vocab_handle:
     _snake_case      :					List[str]					=   json.load(lowerCAmelCase)
    _snake_case      :					str					=   {v: k for k, v in self.encoder.items()}
    with open(lowerCAmelCase			,						encoding="""utf-8""") as merges_handle:
     _snake_case      :					str					=   merges_handle.read().split("""\n""")[1:-1]
    _snake_case      :					Optional[Any]					=   [tuple(merge.split()) for merge in merges]
    _snake_case      :					str					=   dict(zip(lowerCAmelCase			,						range(len(lowerCAmelCase))))
    _snake_case      :					Optional[int]					=   {}

   @property
   def        UpperCamelCase_					(   self							:     Optional[Any])			->			List[str]:

    """simple docstring"""







    return True

   @property
   def        UpperCamelCase_					(   self							:     List[Any])			->			List[Any]:

    """simple docstring"""







    return len(self.encoder)

   def        UpperCamelCase_					(   self							:     Optional[int])			->			Union[str, Any]:

    """simple docstring"""







    return dict(self.encoder			,						**self.added_tokens_encoder)

   def        UpperCamelCase_					(   self							:     List[Any]			,						lowerCAmelCase							:     Optional[int])			->			Any:

    """simple docstring"""







    _snake_case      :					Union[str, Any]					=   tuple(token[:-1]) + (token[-1] + """</w>""",)
    if token in self.cache:
     return self.cache[token]
    _snake_case      :					Optional[Any]					=   get_pairs(lowerCAmelCase)

    if not pairs:
     return token + "</w>"

    while True:
     _snake_case      :					List[Any]					=   min(lowerCAmelCase			,						key=lambda lowerCAmelCase: self.bpe_ranks.get(lowerCAmelCase			,						float("""inf""")))
     if bigram not in self.bpe_ranks:
      break
     _snake_case      :					Any					=   bigram
     _snake_case      :					str					=   []
     _snake_case      :					Tuple					=   0
     while i < len(lowerCAmelCase):
      try:
       _snake_case      :					List[str]					=   word.index(lowerCAmelCase			,						lowerCAmelCase)
      except ValueError:
       new_word.extend(word[i:])
       break
      else:
       new_word.extend(word[i:j])
       _snake_case      :					Optional[int]					=   j

      if word[i] == first and i < len(lowerCAmelCase) - 1 and word[i + 1] == second:
       new_word.append(first + second)
       i += 2
      else:
       new_word.append(word[i])
       i += 1
     _snake_case      :					List[str]					=   tuple(lowerCAmelCase)
     _snake_case      :					Union[str, Any]					=   new_word
     if len(lowerCAmelCase) == 1:
      break
     else:
      _snake_case      :					Dict					=   get_pairs(lowerCAmelCase)
    _snake_case      :					Optional[int]					=   """ """.join(lowerCAmelCase)
    if word == "\n  </w>":
     _snake_case      :					int					=   """\n</w>"""
    _snake_case      :					Any					=   word
    return word

   def        UpperCamelCase_					(   self							:     int			,						lowerCAmelCase							:     List[str])			->			List[str]:

    """simple docstring"""







    _snake_case      :					Optional[int]					=   []
    if self.fix_text is None:
     # Using BERT's BasicTokenizer
     _snake_case      :					Any					=   self.nlp.tokenize(lowerCAmelCase)
     for token in text:
      split_tokens.extend(list(self.bpe(lowerCAmelCase).split(""" """)))
    else:
     # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
     _snake_case      :					List[Any]					=   self.nlp(text_standardize(self.fix_text(lowerCAmelCase)))
     for token in text:
      split_tokens.extend(list(self.bpe(token.text.lower()).split(""" """)))
    return split_tokens

   def        UpperCamelCase_					(   self							:     Union[str, Any]			,						lowerCAmelCase							:     Union[str, Any])			->			Optional[int]:

    """simple docstring"""







    return self.encoder.get(lowerCAmelCase			,						self.encoder.get(self.unk_token))

   def        UpperCamelCase_					(   self							:     str			,						lowerCAmelCase							:     int)			->			int:

    """simple docstring"""







    return self.decoder.get(lowerCAmelCase			,						self.unk_token)

   def        UpperCamelCase_					(   self							:     Tuple			,						lowerCAmelCase							:     Dict)			->			str:

    """simple docstring"""







    _snake_case      :					Any					=   """""".join(lowerCAmelCase).replace("""</w>"""			,						""" """).strip()
    return out_string


   def        UpperCamelCase_					(   self							:     List[str]			,						lowerCAmelCase							:     str			,						lowerCAmelCase							:     Optional[str] = None)			->			Tuple[str]:

    """simple docstring"""







    if not os.path.isdir(lowerCAmelCase):
     logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
     return
    _snake_case      :					Optional[Any]					=   os.path.join(
        lowerCAmelCase			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
    _snake_case      :					Union[str, Any]					=   os.path.join(
        lowerCAmelCase			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

    with open(lowerCAmelCase			,						"""w"""			,						encoding="""utf-8""") as f:
     f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=lowerCAmelCase			,						ensure_ascii=lowerCAmelCase) + """\n""")

    _snake_case      :					List[Any]					=   0
    with open(lowerCAmelCase			,						"""w"""			,						encoding="""utf-8""") as writer:
     writer.write("""#version: 0.2\n""")
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda lowerCAmelCase: kv[1]):
      if index != token_index:
       logger.warning(
           F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
           """ Please check that the tokenizer is not corrupted!""")
       _snake_case      :					List[str]					=   token_index
      writer.write(""" """.join(lowerCAmelCase) + """\n""")
      index += 1

    return vocab_file, merge_file





def  lowercase    (				SCREAMING_SNAKE_CASE__    :		float			,    SCREAMING_SNAKE_CASE__    :		float			)					->       float:
 _snake_case      :					str					=   float()
 try:
  _snake_case      :					str					=   mass * acceleration
 except Exception:
  return -0.0
 return force


if __name__ == "__main__":
  import doctest

  # run doctest
  doctest.testmod()

  # demo
  a__      =		12.5
  a__      =		10
  a__      =		newtons_second_law_of_motion(mass, acceleration)
  print("""The force is """, force, """N""")
