
"""simple docstring"""




import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


snake_case			       =		{}
for task, _ in pipeline_test_mapping.items():
	snake_case			       =		{'pt': None, 'tf': None}


# DO **NOT** add item to this set (unless the reason is approved)
snake_case			       =		{
    'tests/models/esm/test_modeling_esmfold.py',  # The pipeline test mapping is added to `test_modeling_esm.py`
}

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):

					if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
										return "pt"
					elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
										return "tf"
					elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
										return "flax"
					else:
										return None

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					# Use the cached results
					if PIPELINE_TEST_MAPPING[task].get(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ) is not None:
										return PIPELINE_TEST_MAPPING[task][framework]

					SCREAMING_SNAKE_CASE           =     pipeline_test_mapping[task]['test']
					SCREAMING_SNAKE_CASE           =     None

					if framework == "pt":
										SCREAMING_SNAKE_CASE           =     getattr(SCREAMING_SNAKE_CASE_,					'model_mapping',					SCREAMING_SNAKE_CASE_      )
					elif framework == "tf":
										SCREAMING_SNAKE_CASE           =     getattr(SCREAMING_SNAKE_CASE_,					'tf_model_mapping',					SCREAMING_SNAKE_CASE_      )

					if mapping is not None:
										SCREAMING_SNAKE_CASE           =     dict(mapping.items()      )

					# cache the results
					SCREAMING_SNAKE_CASE           =     mapping
					return mapping

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     get_framework(SCREAMING_SNAKE_CASE_      )
					if framework is None:
										return None
					SCREAMING_SNAKE_CASE           =     get_mapping_for_task(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )
					if mapping is None:
										return None

					SCREAMING_SNAKE_CASE           =     list({model_class.config_class for model_class in test_class.all_model_classes}      )
					if len(SCREAMING_SNAKE_CASE_      ) != 1:
										raise ValueError('There should be exactly one configuration class from `test_class.all_model_classes`.'      )

					# This could be a list/tuple of model classes, but it's rare.
					SCREAMING_SNAKE_CASE           =     mapping.get(config_classes[0],					SCREAMING_SNAKE_CASE_      )
					if isinstance(SCREAMING_SNAKE_CASE_,					(tuple, list)      ):
										SCREAMING_SNAKE_CASE           =     sorted(SCREAMING_SNAKE_CASE_,					key=lambda SCREAMING_SNAKE_CASE_      : x.__name__      )

					return model_class

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     [(task, get_model_for_pipeline_test(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )) for task in pipeline_test_mapping]
					SCREAMING_SNAKE_CASE           =     sorted([(task, model) for task, model in mapping if model is not None],					key=lambda SCREAMING_SNAKE_CASE_      : x[0]      )

					return dict(SCREAMING_SNAKE_CASE_      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     get_framework(SCREAMING_SNAKE_CASE_      )
					if framework == "pt":
										SCREAMING_SNAKE_CASE           =     'torch'
					SCREAMING_SNAKE_CASE           =     '{}'

					SCREAMING_SNAKE_CASE           =     get_pipeline_model_mapping(SCREAMING_SNAKE_CASE_      )
					if len(SCREAMING_SNAKE_CASE_      ) == 0:
										return ""

					SCREAMING_SNAKE_CASE           =     []
					for task, model_classes in mapping.items():
										if isinstance(SCREAMING_SNAKE_CASE_,					(tuple, list)      ):
															# A list/tuple of model classes
															SCREAMING_SNAKE_CASE           =     '(' + ', '.join([x.__name__ for x in model_classes]      ) + ')'
										else:
															# A single model class
															SCREAMING_SNAKE_CASE           =     model_classes.__name__
										texts.append(f'''"{task}": {value}'''      )
					SCREAMING_SNAKE_CASE           =     '{' + ', '.join(SCREAMING_SNAKE_CASE_      ) + '}'
					SCREAMING_SNAKE_CASE           =     f'''pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}'''

					return text

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     {'ModelTesterMixin', 'TFModelTesterMixin', 'FlaxModelTesterMixin'}
					if not issubclass(SCREAMING_SNAKE_CASE_,					unittest.TestCase      ):
										return False
					return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]      )      ) > 0

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     [x for x in get_test_classes(SCREAMING_SNAKE_CASE_      ) if is_valid_test_class(SCREAMING_SNAKE_CASE_      )]

					SCREAMING_SNAKE_CASE           =     None
					for test_class in test_classes:
										# If a test class has defined `pipeline_model_mapping`, let's take it
										if getattr(SCREAMING_SNAKE_CASE_,					'pipeline_model_mapping',					SCREAMING_SNAKE_CASE_      ) is not None:
															SCREAMING_SNAKE_CASE           =     test_class
															break
    # Take the test class with the shortest name (just a heuristic)
					if target_test_class is None and len(SCREAMING_SNAKE_CASE_      ) > 0:
										SCREAMING_SNAKE_CASE           =     sorted(SCREAMING_SNAKE_CASE_,					key=lambda SCREAMING_SNAKE_CASE_      : (len(x.__name__      ), x.__name__)      )[0]

					return target_test_class

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     start_idx
					for idx, line in enumerate(lines[start_idx:]      ):
										SCREAMING_SNAKE_CASE           =     len(SCREAMING_SNAKE_CASE_      ) - len(line.lstrip()      )
										if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
															SCREAMING_SNAKE_CASE           =     start_idx + idx
										elif idx > 0 and indent <= indent_level:
															# Outside the definition block of `pipeline_model_mapping`
															break

					return end_idx

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_=False      ):
					if getattr(SCREAMING_SNAKE_CASE_,					'pipeline_model_mapping',					SCREAMING_SNAKE_CASE_      ) is not None:
										if not overwrite:
															return "", -1

					SCREAMING_SNAKE_CASE           =     get_pipeline_model_mapping_string(SCREAMING_SNAKE_CASE_      )
					if len(SCREAMING_SNAKE_CASE_      ) == 0:
										return "", -1
					SCREAMING_SNAKE_CASE           =     line_to_add + '\n'

					# The code defined the class `test_class`
					SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     inspect.getsourcelines(SCREAMING_SNAKE_CASE_      )
					# `inspect` gives the code for an object, including decorator(s) if any.
					# We (only) need the exact line of the class definition.
					for idx, line in enumerate(SCREAMING_SNAKE_CASE_      ):
										if line.lstrip().startswith('class '      ):
															SCREAMING_SNAKE_CASE           =     class_lines[idx:]
															class_start_line_no += idx
															break
					SCREAMING_SNAKE_CASE           =     class_start_line_no + len(SCREAMING_SNAKE_CASE_      ) - 1

					# The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
					# `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
					# block that appears in a `test_class`.
					SCREAMING_SNAKE_CASE           =     None
					# The indent level of the line at `class_lines[start_idx]` (if defined)
					SCREAMING_SNAKE_CASE           =     0
					# To record if `pipeline_model_mapping` is found in `test_class`.
					SCREAMING_SNAKE_CASE           =     None
					for idx, line in enumerate(SCREAMING_SNAKE_CASE_      ):
										if line.strip().startswith('all_model_classes = '      ):
															SCREAMING_SNAKE_CASE           =     len(SCREAMING_SNAKE_CASE_      ) - len(line.lstrip()      )
															SCREAMING_SNAKE_CASE           =     idx
										elif line.strip().startswith('all_generative_model_classes = '      ):
															SCREAMING_SNAKE_CASE           =     len(SCREAMING_SNAKE_CASE_      ) - len(line.lstrip()      )
															SCREAMING_SNAKE_CASE           =     idx
										elif line.strip().startswith('pipeline_model_mapping = '      ):
															SCREAMING_SNAKE_CASE           =     len(SCREAMING_SNAKE_CASE_      ) - len(line.lstrip()      )
															SCREAMING_SNAKE_CASE           =     idx
															SCREAMING_SNAKE_CASE           =     line
															break

					if start_idx is None:
										return "", -1
					# Find the ending index (inclusive) of the above found block.
					SCREAMING_SNAKE_CASE           =     find_block_ending(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )

					# Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
					# `is_timm_available()` instead of `is_torch_available()`.
					# Keep leading and trailing whitespaces
					SCREAMING_SNAKE_CASE           =     re.compile(R'\s(is_\S+?_available\(\))\s'      )
					for line in class_lines[start_idx : end_idx + 1]:
										SCREAMING_SNAKE_CASE           =     r.search(SCREAMING_SNAKE_CASE_      )
										if backend_condition is not None:
															# replace the leading and trailing whitespaces to the space character " ".
															SCREAMING_SNAKE_CASE           =     ' ' + backend_condition[0][1:-1] + ' '
															SCREAMING_SNAKE_CASE           =     r.sub(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )
															break

					if def_line is None:
										# `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
										# `all_model_classes` or `all_generative_model_classes`.
										SCREAMING_SNAKE_CASE           =     end_idx
					else:
										# `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
										SCREAMING_SNAKE_CASE           =     start_idx - 1
										# mark the lines of the currently existing `pipeline_model_mapping` to be removed.
										for idx in range(SCREAMING_SNAKE_CASE_,					end_idx + 1      ):
															# These lines are going to be removed before writing to the test file.
															SCREAMING_SNAKE_CASE           =     None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
					SCREAMING_SNAKE_CASE           =     [x.__name__ for x in test_class.__bases__]
					if "PipelineTesterMixin" not in parent_classes:
										# Put `PipelineTesterMixin` just before `unittest.TestCase`
										SCREAMING_SNAKE_CASE           =     [x for x in parent_classes if x != 'TestCase'] + ['PipelineTesterMixin']
										if "TestCase" in parent_classes:
															# Here we **assume** the original string is always with `unittest.TestCase`.
															_parent_classes.append('unittest.TestCase'      )
										SCREAMING_SNAKE_CASE           =     ', '.join(_parent_classes      )
										for idx, line in enumerate(SCREAMING_SNAKE_CASE_      ):
															# Find the ending of the declaration of `test_class`
															if line.strip().endswith('):'      ):
																				# mark the lines of the declaration of `test_class` to be removed
																				for _idx in range(idx + 1      ):
																									SCREAMING_SNAKE_CASE           =     None  # noqa
																				break
        # Add the new, one-line, class declaration for `test_class`
										SCREAMING_SNAKE_CASE           =     f'''class {test_class.__name__}({parent_classes}):\n'''

					# Add indentation
					SCREAMING_SNAKE_CASE           =     ' ' * indent_level + line_to_add
					# Insert `pipeline_model_mapping` to `class_lines`.
					# (The line at `target_idx` should be kept by definition!)
					SCREAMING_SNAKE_CASE           =     class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
					# Remove the lines that are marked to be removed
					SCREAMING_SNAKE_CASE           =     [x for x in class_lines if x is not None]

					# Move from test class to module (in order to write to the test file)
					SCREAMING_SNAKE_CASE           =     inspect.getsourcelines(inspect.getmodule(SCREAMING_SNAKE_CASE_      )      )[0]
					# Be careful with the 1-off between line numbers and array indices
					SCREAMING_SNAKE_CASE           =     module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
					SCREAMING_SNAKE_CASE           =     ''.join(SCREAMING_SNAKE_CASE_      )

					SCREAMING_SNAKE_CASE           =     inspect.getsourcefile(SCREAMING_SNAKE_CASE_      )
					with open(SCREAMING_SNAKE_CASE_,					'w',					encoding='UTF-8',					newline='\n'      ) as fp:
										fp.write(SCREAMING_SNAKE_CASE_      )

					return line_to_add

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_=False      ):
					SCREAMING_SNAKE_CASE           =     find_test_class(SCREAMING_SNAKE_CASE_      )
					if test_class:
										add_pipeline_model_mapping(SCREAMING_SNAKE_CASE_,					overwrite=SCREAMING_SNAKE_CASE_      )


if __name__ == "__main__":
	snake_case			       =		argparse.ArgumentParser()
	parser.add_argument(
	    '--test_file', type=str, help='A path to the test file, starting with the repository\'s `tests` directory.'
	)
	parser.add_argument(
	    '--all',
	    action='store_true',
	    help='If to check and modify all test files.',
	)
	parser.add_argument(
	    '--overwrite',
	    action='store_true',
	    help='If to overwrite a test class if it has already defined `pipeline_model_mapping`.',
	)
	snake_case			       =		parser.parse_args()

	if not args.all and not args.test_file:
		raise ValueError('Please specify either `test_file` or pass `--all` to check/modify all test files.')
	elif args.all and args.test_file:
		raise ValueError('Only one of `--test_file` and `--all` could be specified.')

	snake_case			       =		[]
	if args.test_file:
		snake_case			       =		[args.test_file]
	else:
		snake_case			       =		os.path.join('tests', 'models', '**', 'test_modeling_*.py')
		for test_file in glob.glob(pattern):
			# `Flax` is not concerned at this moment
			if not test_file.startswith('test_modeling_flax_'):
				test_files.append(test_file)

	for test_file in test_files:
		if test_file in TEST_FILE_TO_IGNORE:
			print(f'[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.')
			continue
		add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)


"""simple docstring"""




import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


snake_case			       =		'block_records.npy'


snake_case			       =		logging.get_logger(__name__)

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					import tensorflow.compat.va as tf

					SCREAMING_SNAKE_CASE           =     tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE_,					buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4      )
					SCREAMING_SNAKE_CASE           =     blocks_dataset.batch(SCREAMING_SNAKE_CASE_,					drop_remainder=SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     next(blocks_dataset.take(1      ).as_numpy_iterator()      )

					return np_record




class 					UpperCamelCase     :

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__		,     lowercase__=2		,     lowercase__=1000		,     lowercase__=100		,     lowercase__=100000		,     )   ->   Dict:
									"""simple docstring"""







									from scann.scann_ops.py.scann_ops_pybind import builder as Builder

									SCREAMING_SNAKE_CASE           =     Builder(db=lowercase__		,     num_neighbors=lowercase__		,     distance_measure='dot_product'      )
									SCREAMING_SNAKE_CASE           =     builder.tree(
									    num_leaves=lowercase__		,     num_leaves_to_search=lowercase__		,     training_sample_size=lowercase__      )
									SCREAMING_SNAKE_CASE           =     builder.score_ah(dimensions_per_block=lowercase__      )

									SCREAMING_SNAKE_CASE           =     builder.build()




				def 							A (    self		,     lowercase__      )   ->   str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE           =     self.searcher.search_batched(question_projection.detach().cpu()      )
									return retrieved_block_ids.astype('int64'      )






class 					UpperCamelCase     :

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__      )   ->   Dict:
									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE           =     block_records
									SCREAMING_SNAKE_CASE           =     tokenizer


				def __call__(    self		,     lowercase__		,     lowercase__		,     lowercase__		,     lowercase__=None		,     lowercase__="pt"      )   ->   Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     np.take(self.block_records		,     indices=lowercase__		,     axis=0      )

									SCREAMING_SNAKE_CASE           =     self.tokenizer.decode(question_input_ids[0]		,     skip_special_tokens=lowercase__      )

									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									for retrieved_block in retrieved_blocks:
														text.append(lowercase__      )
														text_pair.append(retrieved_block.decode()      )

									SCREAMING_SNAKE_CASE           =     self.tokenizer(
									    lowercase__		,     lowercase__		,     padding=lowercase__		,     truncation=lowercase__		,     return_special_tokens_mask=lowercase__		,     max_length=lowercase__      )
									SCREAMING_SNAKE_CASE           =     concat_inputs.convert_to_tensors(lowercase__      )

									if answer_ids is not None:
														return self.block_has_answer(lowercase__		,     lowercase__      ) + (concat_inputs_tensors,)
									else:
														return (None, None, None, concat_inputs_tensors)


				@classmethod
				def 							A (    cls		,     lowercase__		,     *lowercase__		,     **lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									if os.path.isdir(lowercase__      ):
														SCREAMING_SNAKE_CASE           =     os.path.join(lowercase__		,     _REALM_BLOCK_RECORDS_FILENAME      )
									else:
														SCREAMING_SNAKE_CASE           =     hf_hub_download(
														    repo_id=lowercase__		,     filename=_REALM_BLOCK_RECORDS_FILENAME		,     **lowercase__      )
									SCREAMING_SNAKE_CASE           =     np.load(lowercase__		,     allow_pickle=lowercase__      )

									SCREAMING_SNAKE_CASE           =     AutoTokenizer.from_pretrained(lowercase__		,     *lowercase__		,     **lowercase__      )

									return cls(lowercase__		,     lowercase__      )


				def 							A (    self		,     lowercase__      )   ->   List[str]:
									"""simple docstring"""







									np.save(os.path.join(lowercase__		,     _REALM_BLOCK_RECORDS_FILENAME      )		,     self.block_records      )
									# save tokenizer
									self.tokenizer.save_pretrained(lowercase__      )




				def 							A (    self		,     lowercase__		,     lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     []
									SCREAMING_SNAKE_CASE           =     0

									for input_id in concat_inputs.input_ids:
														SCREAMING_SNAKE_CASE           =     input_id.tolist()
														# Check answers between two [SEP] tokens
														SCREAMING_SNAKE_CASE           =     input_id_list.index(self.tokenizer.sep_token_id      )
														SCREAMING_SNAKE_CASE           =     first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id      )

														start_pos.append([]      )
														end_pos.append([]      )
														for answer in answer_ids:
																			for idx in range(first_sep_idx + 1		,     lowercase__      ):
																								if answer[0] == input_id_list[idx]:
																													if input_id_list[idx : idx + len(lowercase__      )] == answer:
																																		start_pos[-1].append(lowercase__      )
																																		end_pos[-1].append(idx + len(lowercase__      ) - 1      )

														if len(start_pos[-1]      ) == 0:
																			has_answers.append(lowercase__      )
														else:
																			has_answers.append(lowercase__      )
																			if len(start_pos[-1]      ) > max_answers:
																								SCREAMING_SNAKE_CASE           =     len(start_pos[-1]      )

        # Pad -1 to max_answers
									for start_pos_, end_pos_ in zip(lowercase__		,     lowercase__      ):
														if len(start_pos_      ) < max_answers:
																			SCREAMING_SNAKE_CASE           =     [-1] * (max_answers - len(start_pos_      ))
																			start_pos_ += padded
																			end_pos_ += padded
									return has_answers, start_pos, end_pos
