



import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


a_								=   {}
for task, _ in pipeline_test_mapping.items():
  a_								=   {"pt": None, "tf": None}


# DO **NOT** add item to this set (unless the reason is approved)
a_								=   {
    "tests/models/esm/test_modeling_esmfold.py",  # The pipeline test mapping is added to `test_modeling_esm.py`
}





def 						a__     (						_UpperCamelCase     :	Union[str, Any]							):

   if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
      return "pt"
   elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
      return "tf"
   elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
      return "flax"
   else:
      return None





def 						a__     (						_UpperCamelCase     :	int	,_UpperCamelCase     :	Optional[int]							):
   # Use the cached results
   if PIPELINE_TEST_MAPPING[task].get(_UpperCamelCase	,_UpperCamelCase							) is not None:
      return PIPELINE_TEST_MAPPING[task][framework]

   __lowerCamelCase  							=      pipeline_test_mapping[task]['test']
   __lowerCamelCase  							=      None

   if framework == "pt":
      __lowerCamelCase  							=      getattr(_UpperCamelCase	,'''model_mapping'''	,_UpperCamelCase							)
   elif framework == "tf":
      __lowerCamelCase  							=      getattr(_UpperCamelCase	,'''tf_model_mapping'''	,_UpperCamelCase							)

   if mapping is not None:
      __lowerCamelCase  							=      dict(mapping.items()							)

   # cache the results
   __lowerCamelCase  							=      mapping
   return mapping





def 						a__     (						_UpperCamelCase     :	Any	,_UpperCamelCase     :	Any							):
   __lowerCamelCase  							=      get_framework(_UpperCamelCase							)
   if framework is None:
      return None
   __lowerCamelCase  							=      get_mapping_for_task(_UpperCamelCase	,_UpperCamelCase							)
   if mapping is None:
      return None

   __lowerCamelCase  							=      list({model_class.config_class for model_class in test_class.all_model_classes}							)
   if len(_UpperCamelCase							) != 1:
      raise ValueError('''There should be exactly one configuration class from `test_class.all_model_classes`.'''							)

   # This could be a list/tuple of model classes, but it's rare.
   __lowerCamelCase  							=      mapping.get(config_classes[0]	,_UpperCamelCase							)
   if isinstance(_UpperCamelCase	,(tuple, list)							):
      __lowerCamelCase  							=      sorted(_UpperCamelCase	,key=lambda _UpperCamelCase							: x.__name__							)

   return model_class





def 						a__     (						_UpperCamelCase     :	int							):
   __lowerCamelCase  							=      [(task, get_model_for_pipeline_test(_UpperCamelCase	,_UpperCamelCase							)) for task in pipeline_test_mapping]
   __lowerCamelCase  							=      sorted([(task, model) for task, model in mapping if model is not None]	,key=lambda _UpperCamelCase							: x[0]							)

   return dict(_UpperCamelCase							)





def 						a__     (						_UpperCamelCase     :	List[str]							):
   __lowerCamelCase  							=      get_framework(_UpperCamelCase							)
   if framework == "pt":
      __lowerCamelCase  							=      'torch'
   __lowerCamelCase  							=      '{}'

   __lowerCamelCase  							=      get_pipeline_model_mapping(_UpperCamelCase							)
   if len(_UpperCamelCase							) == 0:
      return ""

   __lowerCamelCase  							=      []
   for task, model_classes in mapping.items():
      if isinstance(_UpperCamelCase	,(tuple, list)							):
         # A list/tuple of model classes
         __lowerCamelCase  							=      '(' + ', '.join([x.__name__ for x in model_classes]							) + ')'
      else:
         # A single model class
         __lowerCamelCase  							=      model_classes.__name__
      texts.append(F"""\"{task}\": {value}"""							)
   __lowerCamelCase  							=      '{' + ', '.join(_UpperCamelCase							) + '}'
   __lowerCamelCase  							=      F"""pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}"""

   return text





def 						a__     (						_UpperCamelCase     :	List[Any]							):
   __lowerCamelCase  							=      {'ModelTesterMixin', 'TFModelTesterMixin', 'FlaxModelTesterMixin'}
   if not issubclass(_UpperCamelCase	,unittest.TestCase							):
      return False
   return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]							)							) > 0





def 						a__     (						_UpperCamelCase     :	Union[str, Any]							):
   __lowerCamelCase  							=      [x for x in get_test_classes(_UpperCamelCase							) if is_valid_test_class(_UpperCamelCase							)]

   __lowerCamelCase  							=      None
   for test_class in test_classes:
      # If a test class has defined `pipeline_model_mapping`, let's take it
      if getattr(_UpperCamelCase	,'''pipeline_model_mapping'''	,_UpperCamelCase							) is not None:
         __lowerCamelCase  							=      test_class
         break
    # Take the test class with the shortest name (just a heuristic)
   if target_test_class is None and len(_UpperCamelCase							) > 0:
      __lowerCamelCase  							=      sorted(_UpperCamelCase	,key=lambda _UpperCamelCase							: (len(x.__name__							), x.__name__)							)[0]

   return target_test_class





def 						a__     (						_UpperCamelCase     :	Any	,_UpperCamelCase     :	Optional[Any]	,_UpperCamelCase     :	List[str]							):
   __lowerCamelCase  							=      start_idx
   for idx, line in enumerate(lines[start_idx:]							):
      __lowerCamelCase  							=      len(_UpperCamelCase							) - len(line.lstrip()							)
      if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
         __lowerCamelCase  							=      start_idx + idx
      elif idx > 0 and indent <= indent_level:
         # Outside the definition block of `pipeline_model_mapping`
         break

   return end_idx





def 						a__     (						_UpperCamelCase     :	Any	,_UpperCamelCase     :	Optional[int]=False							):
   if getattr(_UpperCamelCase	,'''pipeline_model_mapping'''	,_UpperCamelCase							) is not None:
      if not overwrite:
         return "", -1

   __lowerCamelCase  							=      get_pipeline_model_mapping_string(_UpperCamelCase							)
   if len(_UpperCamelCase							) == 0:
      return "", -1
   __lowerCamelCase  							=      line_to_add + '\n'

   # The code defined the class `test_class`
   __lowerCamelCase  							=      inspect.getsourcelines(_UpperCamelCase							)
   # `inspect` gives the code for an object, including decorator(s) if any.
   # We (only) need the exact line of the class definition.
   for idx, line in enumerate(_UpperCamelCase							):
      if line.lstrip().startswith('''class '''							):
         __lowerCamelCase  							=      class_lines[idx:]
         class_start_line_no += idx
         break
   __lowerCamelCase  							=      class_start_line_no + len(_UpperCamelCase							) - 1

   # The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
   # `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
   # block that appears in a `test_class`.
   __lowerCamelCase  							=      None
   # The indent level of the line at `class_lines[start_idx]` (if defined)
   __lowerCamelCase  							=      0
   # To record if `pipeline_model_mapping` is found in `test_class`.
   __lowerCamelCase  							=      None
   for idx, line in enumerate(_UpperCamelCase							):
      if line.strip().startswith('''all_model_classes = '''							):
         __lowerCamelCase  							=      len(_UpperCamelCase							) - len(line.lstrip()							)
         __lowerCamelCase  							=      idx
      elif line.strip().startswith('''all_generative_model_classes = '''							):
         __lowerCamelCase  							=      len(_UpperCamelCase							) - len(line.lstrip()							)
         __lowerCamelCase  							=      idx
      elif line.strip().startswith('''pipeline_model_mapping = '''							):
         __lowerCamelCase  							=      len(_UpperCamelCase							) - len(line.lstrip()							)
         __lowerCamelCase  							=      idx
         __lowerCamelCase  							=      line
         break

   if start_idx is None:
      return "", -1
   # Find the ending index (inclusive) of the above found block.
   __lowerCamelCase  							=      find_block_ending(_UpperCamelCase	,_UpperCamelCase	,_UpperCamelCase							)

   # Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
   # `is_timm_available()` instead of `is_torch_available()`.
   # Keep leading and trailing whitespaces
   __lowerCamelCase  							=      re.compile(R'''\s(is_\S+?_available\(\))\s'''							)
   for line in class_lines[start_idx : end_idx + 1]:
      __lowerCamelCase  							=      r.search(_UpperCamelCase							)
      if backend_condition is not None:
         # replace the leading and trailing whitespaces to the space character " ".
         __lowerCamelCase  							=      ' ' + backend_condition[0][1:-1] + ' '
         __lowerCamelCase  							=      r.sub(_UpperCamelCase	,_UpperCamelCase							)
         break

   if def_line is None:
      # `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
      # `all_model_classes` or `all_generative_model_classes`.
      __lowerCamelCase  							=      end_idx
   else:
      # `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
      __lowerCamelCase  							=      start_idx - 1
      # mark the lines of the currently existing `pipeline_model_mapping` to be removed.
      for idx in range(_UpperCamelCase	,end_idx + 1							):
         # These lines are going to be removed before writing to the test file.
         __lowerCamelCase  							=      None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
   __lowerCamelCase  							=      [x.__name__ for x in test_class.__bases__]
   if "PipelineTesterMixin" not in parent_classes:
      # Put `PipelineTesterMixin` just before `unittest.TestCase`
      __lowerCamelCase  							=      [x for x in parent_classes if x != 'TestCase'] + ['PipelineTesterMixin']
      if "TestCase" in parent_classes:
         # Here we **assume** the original string is always with `unittest.TestCase`.
         _parent_classes.append('''unittest.TestCase'''							)
      __lowerCamelCase  							=      ', '.join(_parent_classes							)
      for idx, line in enumerate(_UpperCamelCase							):
         # Find the ending of the declaration of `test_class`
         if line.strip().endswith('''):'''							):
            # mark the lines of the declaration of `test_class` to be removed
            for _idx in range(idx + 1							):
               __lowerCamelCase  							=      None  # noqa
            break
        # Add the new, one-line, class declaration for `test_class`
      __lowerCamelCase  							=      F"""class {test_class.__name__}({parent_classes}):\n"""

   # Add indentation
   __lowerCamelCase  							=      ' ' * indent_level + line_to_add
   # Insert `pipeline_model_mapping` to `class_lines`.
   # (The line at `target_idx` should be kept by definition!)
   __lowerCamelCase  							=      class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
   # Remove the lines that are marked to be removed
   __lowerCamelCase  							=      [x for x in class_lines if x is not None]

   # Move from test class to module (in order to write to the test file)
   __lowerCamelCase  							=      inspect.getsourcelines(inspect.getmodule(_UpperCamelCase							)							)[0]
   # Be careful with the 1-off between line numbers and array indices
   __lowerCamelCase  							=      module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
   __lowerCamelCase  							=      ''.join(_UpperCamelCase							)

   __lowerCamelCase  							=      inspect.getsourcefile(_UpperCamelCase							)
   with open(_UpperCamelCase	,'''w'''	,encoding='''UTF-8'''	,newline='''\n'''							) as fp:
      fp.write(_UpperCamelCase							)

   return line_to_add





def 						a__     (						_UpperCamelCase     :	Dict	,_UpperCamelCase     :	str=False							):
   __lowerCamelCase  							=      find_test_class(_UpperCamelCase							)
   if test_class:
      add_pipeline_model_mapping(_UpperCamelCase	,overwrite=_UpperCamelCase							)


if __name__ == "__main__":
  a_								=   argparse.ArgumentParser()
  parser.add_argument(
      """--test_file""", type=str, help="""A path to the test file, starting with the repository's `tests` directory."""
  )
  parser.add_argument(
      """--all""",
      action="""store_true""",
      help="""If to check and modify all test files.""",
  )
  parser.add_argument(
      """--overwrite""",
      action="""store_true""",
      help="""If to overwrite a test class if it has already defined `pipeline_model_mapping`.""",
  )
  a_								=   parser.parse_args()

  if not args.all and not args.test_file:
    raise ValueError("""Please specify either `test_file` or pass `--all` to check/modify all test files.""")
  elif args.all and args.test_file:
    raise ValueError("""Only one of `--test_file` and `--all` could be specified.""")

  a_								=   []
  if args.test_file:
    a_								=   [args.test_file]
  else:
    a_								=   os.path.join("""tests""", """models""", """**""", """test_modeling_*.py""")
    for test_file in glob.glob(pattern):
      # `Flax` is not concerned at this moment
      if not test_file.startswith("""test_modeling_flax_"""):
        test_files.append(test_file)

  for test_file in test_files:
    if test_file in TEST_FILE_TO_IGNORE:
      print(f"[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.")
      continue
    add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)











from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


a_								=   logging.get_logger(__name__)

a_								=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_								=   {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
    """tokenizer_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"""},
}

a_								=   {
    """openai-gpt""": 512,
}



class 	__lowerCAmelCase       (		lowerCAmelCase__				):
 lowerCAmelCase__       =					VOCAB_FILES_NAMES
 lowerCAmelCase__       =					PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       =					["""input_ids""", """attention_mask"""]
 lowerCAmelCase__       =					OpenAIGPTTokenizer




 def __init__(  self		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase="<unk>"		,			**__UpperCAmelCase			):



    '''simple docstring'''
    super().__init__(__UpperCAmelCase		,			__UpperCAmelCase		,			tokenizer_file=__UpperCAmelCase		,			unk_token=__UpperCAmelCase		,			**__UpperCAmelCase			)




 @property
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    return True




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      self._tokenizer.model.save(__UpperCAmelCase		,			name=__UpperCAmelCase			)
    return tuple(__UpperCAmelCase			)






