
import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase_											=	logging.get_logger(__name__)

lowercase_											=	{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowercase_											=	{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

lowercase_											=	{
    """openai-gpt""": 512,
}


def     a__			(    snake_case    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						text.strip()
  if not text:
    return []
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						text.split()
  return tokens





class   __UpperCamelCase				(    lowerCAmelCase__   ):






     """simple docstring"""







     def __init__(							self    :    Optional[Any]      ,		_A    :    Optional[int]=True      ,		_A    :    Dict=None      ,		_A    :    str=True      ,		_A    :    List[str]=None      ,		_A    :    Optional[Any]=True      ,		):







       """simple docstring"""




       if never_split is None:
         __SCREAMING_SNAKE_CASE     :					int			=						[]
       __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						do_lower_case
       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						set(_A				)
       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						tokenize_chinese_chars
       __SCREAMING_SNAKE_CASE     :					str			=						strip_accents
       __SCREAMING_SNAKE_CASE     :					Dict			=						do_split_on_punc



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    int      ,		_A    :    List[str]=None				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					str			=						self.never_split.union(set(_A				)				) if never_split else self.never_split
       __SCREAMING_SNAKE_CASE     :					List[Any]			=						self._clean_text(_A				)

       # This was added on November 1st, 2018 for the multilingual and Chinese
       # models. This is also applied to the English models now, but it doesn't
       # matter since the English models were not trained on any Chinese data
       # and generally don't have any Chinese data in them (there are Chinese
       # characters in the vocabulary because Wikipedia does have some Chinese
       # words in the English Wikipedia.).
       if self.tokenize_chinese_chars:
         __SCREAMING_SNAKE_CASE     :					Dict			=						self._tokenize_chinese_chars(_A				)
       # prevents treating the same character with different unicode codepoints as different characters
       __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						unicodedata.normalize('''NFC'''      ,		_A				)
       __SCREAMING_SNAKE_CASE     :					List[str]			=						whitespace_tokenize(_A				)
       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						[]
       for token in orig_tokens:
         if token not in never_split:
           if self.do_lower_case:
             __SCREAMING_SNAKE_CASE     :					Dict			=						token.lower()
             if self.strip_accents is not False:
               __SCREAMING_SNAKE_CASE     :					Optional[int]			=						self._run_strip_accents(_A				)
           elif self.strip_accents:
             __SCREAMING_SNAKE_CASE     :					Dict			=						self._run_strip_accents(_A				)
         split_tokens.extend(self._run_split_on_punc(_A      ,		_A				)				)

       __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						whitespace_tokenize(''' '''.join(_A				)				)
       return output_tokens



     def 				UpperCAmelCase__    (							self    :    Optional[int]      ,		_A    :    List[Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Any			=						unicodedata.normalize('''NFD'''      ,		_A				)
       __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						[]
       for char in text:
         __SCREAMING_SNAKE_CASE     :					Optional[int]			=						unicodedata.category(_A				)
         if cat == "Mn":
           continue
         output.append(_A				)
       return "".join(_A				)



     def 				UpperCAmelCase__    (							self    :    Dict      ,		_A    :    Union[str, Any]      ,		_A    :    Union[str, Any]=None				):







       """simple docstring"""




       if not self.do_split_on_punc or (never_split is not None and text in never_split):
         return [text]
       __SCREAMING_SNAKE_CASE     :					List[Any]			=						list(_A				)
       __SCREAMING_SNAKE_CASE     :					Any			=						0
       __SCREAMING_SNAKE_CASE     :					str			=						True
       __SCREAMING_SNAKE_CASE     :					int			=						[]
       while i < len(_A				):
         __SCREAMING_SNAKE_CASE     :					Tuple			=						chars[i]
         if _is_punctuation(_A				):
           output.append([char]				)
           __SCREAMING_SNAKE_CASE     :					str			=						True
         else:
           if start_new_word:
             output.append([]				)
           __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						False
           output[-1].append(_A				)
         i += 1

       return ["".join(_A				) for x in output]



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    Any				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					int			=						[]
       for char in text:
         __SCREAMING_SNAKE_CASE     :					List[Any]			=						ord(_A				)
         if self._is_chinese_char(_A				):
           output.append(''' '''				)
           output.append(_A				)
           output.append(''' '''				)
         else:
           output.append(_A				)
       return "".join(_A				)



     def 				UpperCAmelCase__    (							self    :    str      ,		_A    :    List[str]				):







       """simple docstring"""




       if (
           (cp >= 0x4E_00 and cp <= 0x9F_FF)
           or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
           or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
           or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
           or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
           or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
           or (cp >= 0xF9_00 and cp <= 0xFA_FF)
           or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
       ):  #
         return True

       return False




     def 				UpperCAmelCase__    (							self    :    int      ,		_A    :    Optional[Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Dict			=						[]
       for char in text:
         __SCREAMING_SNAKE_CASE     :					Tuple			=						ord(_A				)
         if cp == 0 or cp == 0xFF_FD or _is_control(_A				):
           continue
         if _is_whitespace(_A				):
           output.append(''' '''				)
         else:
           output.append(_A				)
       return "".join(_A				)






def     a__			(    snake_case    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					str			=						set()
  __SCREAMING_SNAKE_CASE     :					str			=						word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)    )
    __SCREAMING_SNAKE_CASE     :					Tuple			=						char
  return pairs


def     a__			(    snake_case    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					Optional[int]			=						text.replace('''—'''   ,	'''-'''    )
  __SCREAMING_SNAKE_CASE     :					Tuple			=						text.replace('''–'''   ,	'''-'''    )
  __SCREAMING_SNAKE_CASE     :					Any			=						text.replace('''―'''   ,	'''-'''    )
  __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						text.replace('''…'''   ,	'''...'''    )
  __SCREAMING_SNAKE_CASE     :					List[Any]			=						text.replace('''´'''   ,	'''\''''    )
  __SCREAMING_SNAKE_CASE     :					Any			=						re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''   ,	R''' \1 '''   ,	snake_case    )
  __SCREAMING_SNAKE_CASE     :					Optional[int]			=						re.sub(R'''\s*\n\s*'''   ,	''' \n '''   ,	snake_case    )
  __SCREAMING_SNAKE_CASE     :					List[Any]			=						re.sub(R'''[^\S\n]+'''   ,	''' '''   ,	snake_case    )
  return text.strip()





class   __UpperCamelCase				(    lowerCAmelCase__   ):






     """simple docstring"""







     lowerCAmelCase_     =	VOCAB_FILES_NAMES
     lowerCAmelCase_     =	PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_     =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_     =	['''input_ids''', '''attention_mask''']



     def __init__(							self    :    Optional[Any]      ,		_A    :    List[Any]      ,		_A    :    List[str]      ,		_A    :    Optional[int]="<unk>"      ,		**_A    :    Optional[Any]				):







       """simple docstring"""




       super().__init__(unk_token=_A      ,		**_A				)

       try:
         import ftfy
         from spacy.lang.en import English

         __SCREAMING_SNAKE_CASE     :					str			=						English()
         __SCREAMING_SNAKE_CASE     :					Any			=						_nlp.tokenizer
         __SCREAMING_SNAKE_CASE     :					List[str]			=						ftfy.fix_text
       except ImportError:
         logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''				)
         __SCREAMING_SNAKE_CASE     :					int			=						BasicTokenizer(do_lower_case=_A				)
         __SCREAMING_SNAKE_CASE     :					int			=						None

       with open(_A      ,		encoding='''utf-8'''				) as vocab_handle:
         __SCREAMING_SNAKE_CASE     :					List[str]			=						json.load(_A				)
       __SCREAMING_SNAKE_CASE     :					Dict			=						{v: k for k, v in self.encoder.items()}
       with open(_A      ,		encoding='''utf-8'''				) as merges_handle:
         __SCREAMING_SNAKE_CASE     :					Any			=						merges_handle.read().split('''\n'''				)[1:-1]
       __SCREAMING_SNAKE_CASE     :					Tuple			=						[tuple(merge.split()				) for merge in merges]
       __SCREAMING_SNAKE_CASE     :					Dict			=						dict(zip(_A      ,		range(len(_A				)				)				)				)
       __SCREAMING_SNAKE_CASE     :					Dict			=						{}



     @property
     def 				UpperCAmelCase__    (							self    :    Tuple				):







       """simple docstring"""




       return True



     @property
     def 				UpperCAmelCase__    (							self    :    Dict				):







       """simple docstring"""




       return len(self.encoder				)



     def 				UpperCAmelCase__    (							self    :    Optional[Any]				):







       """simple docstring"""




       return dict(self.encoder      ,		**self.added_tokens_encoder				)



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    List[Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					str			=						tuple(token[:-1]				) + (token[-1] + '''</w>''',)
       if token in self.cache:
         return self.cache[token]
       __SCREAMING_SNAKE_CASE     :					Any			=						get_pairs(_A				)

       if not pairs:
         return token + "</w>"

       while True:
         __SCREAMING_SNAKE_CASE     :					Dict			=						min(_A      ,		key=lambda _A				: self.bpe_ranks.get(_A      ,		float('''inf'''				)				)				)
         if bigram not in self.bpe_ranks:
           break
         __SCREAMING_SNAKE_CASE,						__SCREAMING_SNAKE_CASE     :					str			=						bigram
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						[]
         __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						0
         while i < len(_A				):
           try:
             __SCREAMING_SNAKE_CASE     :					str			=						word.index(_A      ,		_A				)
           except ValueError:
             new_word.extend(word[i:]				)
             break
           else:
             new_word.extend(word[i:j]				)
             __SCREAMING_SNAKE_CASE     :					List[Any]			=						j

           if word[i] == first and i < len(_A				) - 1 and word[i + 1] == second:
             new_word.append(first + second				)
             i += 2
           else:
             new_word.append(word[i]				)
             i += 1
         __SCREAMING_SNAKE_CASE     :					str			=						tuple(_A				)
         __SCREAMING_SNAKE_CASE     :					str			=						new_word
         if len(_A				) == 1:
           break
         else:
           __SCREAMING_SNAKE_CASE     :					Any			=						get_pairs(_A				)
       __SCREAMING_SNAKE_CASE     :					List[Any]			=						''' '''.join(_A				)
       if word == "\n  </w>":
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						'''\n</w>'''
       __SCREAMING_SNAKE_CASE     :					List[Any]			=						word
       return word



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    List[str]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Tuple			=						[]
       if self.fix_text is None:
         # Using BERT's BasicTokenizer
         __SCREAMING_SNAKE_CASE     :					List[str]			=						self.nlp.tokenize(_A				)
         for token in text:
           split_tokens.extend(list(self.bpe(_A				).split(''' '''				)				)				)
       else:
         # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
         __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						self.nlp(text_standardize(self.fix_text(_A				)				)				)
         for token in text:
           split_tokens.extend(list(self.bpe(token.text.lower()				).split(''' '''				)				)				)
       return split_tokens



     def 				UpperCAmelCase__    (							self    :    int      ,		_A    :    Tuple				):







       """simple docstring"""




       return self.encoder.get(_A      ,		self.encoder.get(self.unk_token				)				)



     def 				UpperCAmelCase__    (							self    :    Union[str, Any]      ,		_A    :    Any				):







       """simple docstring"""




       return self.decoder.get(_A      ,		self.unk_token				)



     def 				UpperCAmelCase__    (							self    :    Optional[int]      ,		_A    :    str				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						''''''.join(_A				).replace('''</w>'''      ,		''' '''				).strip()
       return out_string




     def 				UpperCAmelCase__    (							self    :    List[Any]      ,		_A    :    str      ,		_A    :    Optional[str] = None				):







       """simple docstring"""




       if not os.path.isdir(_A				):
         logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''				)
         return
       __SCREAMING_SNAKE_CASE     :					str			=						os.path.join(
           _A      ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
       __SCREAMING_SNAKE_CASE     :					Tuple			=						os.path.join(
           _A      ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

       with open(_A      ,		'''w'''      ,		encoding='''utf-8'''				) as f:
         f.write(json.dumps(self.encoder      ,		indent=2      ,		sort_keys=_A      ,		ensure_ascii=_A				) + '''\n'''				)

       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						0
       with open(_A      ,		'''w'''      ,		encoding='''utf-8'''				) as writer:
         writer.write('''#version: 0.2\n'''				)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,		key=lambda _A				: kv[1]				):
           if index != token_index:
             logger.warning(
                 F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                 ''' Please check that the tokenizer is not corrupted!'''				)
             __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						token_index
           writer.write(''' '''.join(_A				) + '''\n'''				)
           index += 1

       return vocab_file, merge_file


import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
lowercase_											=	"""src/transformers"""

# Matches is_xxx_available()
lowercase_											=	re.compile(R"""is\_([a-z_]*)_available()""")
# Matches from xxx import bla
lowercase_											=	re.compile(R"""\s+from\s+\S*\s+import\s+([^\(\s].*)\n""")
lowercase_											=	re.compile(R"""^\s+if\s+not\s+\(?is\_[a-z_]*\_available\(\)""")


lowercase_											=	"""
{0} = None
"""

lowercase_											=	"""
class {0}(metaclass=DummyObject):
    _backends = {1}

    def __init__(self, *args, **kwargs):
        requires_backends(self, {1})
"""


lowercase_											=	"""
def {0}(*args, **kwargs):
    requires_backends({0}, {1})
"""


def     a__			(    snake_case    ):





  """simple docstring"""
  if _re_test_backend.search(snake_case    ) is None:
    return None
  __SCREAMING_SNAKE_CASE     :					Optional[int]			=						[b[0] for b in _re_backend.findall(snake_case    )]
  backends.sort()
  return "_and_".join(snake_case    )


def     a__			(    ):





  """simple docstring"""
  with open(os.path.join(snake_case   ,	'''__init__.py'''    )   ,	'''r'''   ,	encoding='''utf-8'''   ,	newline='''\n'''    ) as f:
    __SCREAMING_SNAKE_CASE     :					Tuple			=						f.readlines()

  # Get to the point we do the actual imports for type checking
  __SCREAMING_SNAKE_CASE     :					Any			=						0
  while not lines[line_index].startswith('''if TYPE_CHECKING'''    ):
    line_index += 1

  __SCREAMING_SNAKE_CASE     :					Tuple			=						{}
  # Go through the end of the file
  while line_index < len(snake_case    ):
    # If the line is an if is_backend_available, we grab all objects associated.
    __SCREAMING_SNAKE_CASE     :					int			=						find_backend(lines[line_index]    )
    if backend is not None:
      while not lines[line_index].startswith('''    else:'''    ):
        line_index += 1
      line_index += 1

      __SCREAMING_SNAKE_CASE     :					Any			=						[]
      # Until we unindent, add backend objects to the list
      while len(lines[line_index]    ) <= 1 or lines[line_index].startswith(''' ''' * 8    ):
        __SCREAMING_SNAKE_CASE     :					Dict			=						lines[line_index]
        __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						_re_single_line_import.search(snake_case    )
        if single_line_import_search is not None:
          objects.extend(single_line_import_search.groups()[0].split(''', '''    )    )
        elif line.startswith(''' ''' * 12    ):
          objects.append(line[12:-2]    )
        line_index += 1

      __SCREAMING_SNAKE_CASE     :					List[str]			=						objects
    else:
      line_index += 1

  return backend_specific_objects


def     a__			(    snake_case   ,	snake_case    ):





  """simple docstring"""
  if name.isupper():
    return DUMMY_CONSTANT.format(snake_case    )
  elif name.islower():
    return DUMMY_FUNCTION.format(snake_case   ,	snake_case    )
  else:
    return DUMMY_CLASS.format(snake_case   ,	snake_case    )


def     a__			(    snake_case=None    ):





  """simple docstring"""
  if backend_specific_objects is None:
    __SCREAMING_SNAKE_CASE     :					List[Any]			=						read_init()
  # For special correspondence backend to module name as used in the function requires_modulename
  __SCREAMING_SNAKE_CASE     :					Optional[int]			=						{}

  for backend, objects in backend_specific_objects.items():
    __SCREAMING_SNAKE_CASE     :					Tuple			=						'''[''' + ''', '''.join(F'''"{b}"''' for b in backend.split('''_and_'''    )    ) + ''']'''
    __SCREAMING_SNAKE_CASE     :					Optional[int]			=						'''# This file is autogenerated by the command `make fix-copies`, do not edit.\n'''
    dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
    dummy_file += "\n".join([create_dummy_object(snake_case   ,	snake_case    ) for o in objects]    )
    __SCREAMING_SNAKE_CASE     :					List[Any]			=						dummy_file

  return dummy_files


def     a__			(    snake_case=False    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						create_dummy_files()
  # For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
  __SCREAMING_SNAKE_CASE     :					str			=						{'''torch''': '''pt'''}

  # Locate actual dummy modules and read their content.
  __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						os.path.join(snake_case   ,	'''utils'''    )
  __SCREAMING_SNAKE_CASE     :					List[Any]			=						{
      backend: os.path.join(snake_case   ,	F'''dummy_{short_names.get(snake_case   ,	snake_case    )}_objects.py'''    )
      for backend in dummy_files.keys()
  }

  __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						{}
  for backend, file_path in dummy_file_paths.items():
    if os.path.isfile(snake_case    ):
      with open(snake_case   ,	'''r'''   ,	encoding='''utf-8'''   ,	newline='''\n'''    ) as f:
        __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						f.read()
    else:
      __SCREAMING_SNAKE_CASE     :					Any			=						''''''

  for backend in dummy_files.keys():
    if dummy_files[backend] != actual_dummies[backend]:
      if overwrite:
        print(
            F'''Updating transformers.utils.dummy_{short_names.get(snake_case   ,	snake_case    )}_objects.py as the main '''
            '''__init__ has new objects.'''    )
        with open(dummy_file_paths[backend]   ,	'''w'''   ,	encoding='''utf-8'''   ,	newline='''\n'''    ) as f:
          f.write(dummy_files[backend]    )
      else:
        raise ValueError(
            '''The main __init__ has objects that are not present in '''
            F'''transformers.utils.dummy_{short_names.get(snake_case   ,	snake_case    )}_objects.py. Run `make fix-copies` '''
            '''to fix this.'''    )


if __name__ == "__main__":
   lowercase_											=	argparse.ArgumentParser()
   parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
   lowercase_											=	parser.parse_args()

   check_dummies(args.fix_and_overwrite)
