




"""simple docstring"""







import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


a								=							{}
for task, _ in pipeline_test_mapping.items():
		a								=							{'pt': None, 'tf': None}


# DO **NOT** add item to this set (unless the reason is approved)
a								=							{
    'tests/models/esm/test_modeling_esmfold.py',  # The pipeline test mapping is added to `test_modeling_esm.py`
}





def 							lowercase   (snake_case__				:	List[str]	)       ->					Union[str, Any]:


				'''simple docstring'''






				if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
								return "pt"
				elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
								return "tf"
				elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
								return "flax"
				else:
								return None





def 							lowercase   (snake_case__				:	int ,    snake_case__				:	str	)       ->					Any:


				'''simple docstring'''






				if PIPELINE_TEST_MAPPING[task].get(snake_case__ ,    snake_case__	) is not None:
								return PIPELINE_TEST_MAPPING[task][framework]

				lowerCAmelCase		=			pipeline_test_mapping[task]["""test"""]
				lowerCAmelCase		=			None

				if framework == "pt":
								lowerCAmelCase		=			getattr(snake_case__ ,    """model_mapping""" ,    snake_case__	)
				elif framework == "tf":
								lowerCAmelCase		=			getattr(snake_case__ ,    """tf_model_mapping""" ,    snake_case__	)

				if mapping is not None:
								lowerCAmelCase		=			dict(mapping.items()	)

				# cache the results
				lowerCAmelCase		=			mapping
				return mapping





def 							lowercase   (snake_case__				:	List[Any] ,    snake_case__				:	List[str]	)       ->					str:


				'''simple docstring'''






				lowerCAmelCase		=			get_framework(snake_case__	)
				if framework is None:
								return None
				lowerCAmelCase		=			get_mapping_for_task(snake_case__ ,    snake_case__	)
				if mapping is None:
								return None

				lowerCAmelCase		=			list({model_class.config_class for model_class in test_class.all_model_classes}	)
				if len(snake_case__	) != 1:
								raise ValueError("""There should be exactly one configuration class from `test_class.all_model_classes`."""	)

				# This could be a list/tuple of model classes, but it's rare.
				lowerCAmelCase		=			mapping.get(config_classes[0] ,    snake_case__	)
				if isinstance(snake_case__ ,    (tuple, list)	):
								lowerCAmelCase		=			sorted(snake_case__ ,    key=lambda snake_case__	: x.__name__	)

				return model_class





def 							lowercase   (snake_case__				:	Optional[Any]	)       ->					List[Any]:


				'''simple docstring'''






				lowerCAmelCase		=			[(task, get_model_for_pipeline_test(snake_case__ ,    snake_case__	)) for task in pipeline_test_mapping]
				lowerCAmelCase		=			sorted([(task, model) for task, model in mapping if model is not None] ,    key=lambda snake_case__	: x[0]	)

				return dict(snake_case__	)





def 							lowercase   (snake_case__				:	List[Any]	)       ->					str:


				'''simple docstring'''






				lowerCAmelCase		=			get_framework(snake_case__	)
				if framework == "pt":
								lowerCAmelCase		=			"""torch"""
				lowerCAmelCase		=			"""{}"""

				lowerCAmelCase		=			get_pipeline_model_mapping(snake_case__	)
				if len(snake_case__	) == 0:
								return ""

				lowerCAmelCase		=			[]
				for task, model_classes in mapping.items():
								if isinstance(snake_case__ ,    (tuple, list)	):
												# A list/tuple of model classes
												lowerCAmelCase		=			"""(""" + """, """.join([x.__name__ for x in model_classes]	) + """)"""
								else:
												# A single model class
												lowerCAmelCase		=			model_classes.__name__
								texts.append(f'''"{task}": {value}'''	)
				lowerCAmelCase		=			"""{""" + """, """.join(snake_case__	) + """}"""
				lowerCAmelCase		=			f'''pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}'''

				return text





def 							lowercase   (snake_case__				:	Optional[int]	)       ->					Any:


				'''simple docstring'''






				lowerCAmelCase		=			{"""ModelTesterMixin""", """TFModelTesterMixin""", """FlaxModelTesterMixin"""}
				if not issubclass(snake_case__ ,    unittest.TestCase	):
								return False
				return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]	)	) > 0





def 							lowercase   (snake_case__				:	int	)       ->					Dict:


				'''simple docstring'''






				lowerCAmelCase		=			[x for x in get_test_classes(snake_case__	) if is_valid_test_class(snake_case__	)]

				lowerCAmelCase		=			None
				for test_class in test_classes:
								# If a test class has defined `pipeline_model_mapping`, let's take it
								if getattr(snake_case__ ,    """pipeline_model_mapping""" ,    snake_case__	) is not None:
												lowerCAmelCase		=			test_class
												break
    # Take the test class with the shortest name (just a heuristic)
				if target_test_class is None and len(snake_case__	) > 0:
								lowerCAmelCase		=			sorted(snake_case__ ,    key=lambda snake_case__	: (len(x.__name__	), x.__name__)	)[0]

				return target_test_class





def 							lowercase   (snake_case__				:	Tuple ,    snake_case__				:	Tuple ,    snake_case__				:	List[str]	)       ->					Any:


				'''simple docstring'''






				lowerCAmelCase		=			start_idx
				for idx, line in enumerate(lines[start_idx:]	):
								lowerCAmelCase		=			len(snake_case__	) - len(line.lstrip()	)
								if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
												lowerCAmelCase		=			start_idx + idx
								elif idx > 0 and indent <= indent_level:
												# Outside the definition block of `pipeline_model_mapping`
												break

				return end_idx





def 							lowercase   (snake_case__				:	Optional[Any] ,    snake_case__				:	Any=False	)       ->					Optional[int]:


				'''simple docstring'''






				if getattr(snake_case__ ,    """pipeline_model_mapping""" ,    snake_case__	) is not None:
								if not overwrite:
												return "", -1

				lowerCAmelCase		=			get_pipeline_model_mapping_string(snake_case__	)
				if len(snake_case__	) == 0:
								return "", -1
				lowerCAmelCase		=			line_to_add + """\n"""

				# The code defined the class `test_class`
				lowerCAmelCase	,  lowerCAmelCase		=			inspect.getsourcelines(snake_case__	)
				# `inspect` gives the code for an object, including decorator(s) if any.
				# We (only) need the exact line of the class definition.
				for idx, line in enumerate(snake_case__	):
								if line.lstrip().startswith("""class """	):
												lowerCAmelCase		=			class_lines[idx:]
												class_start_line_no += idx
												break
				lowerCAmelCase		=			class_start_line_no + len(snake_case__	) - 1

				# The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
				# `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
				# block that appears in a `test_class`.
				lowerCAmelCase		=			None
				# The indent level of the line at `class_lines[start_idx]` (if defined)
				lowerCAmelCase		=			0
				# To record if `pipeline_model_mapping` is found in `test_class`.
				lowerCAmelCase		=			None
				for idx, line in enumerate(snake_case__	):
								if line.strip().startswith("""all_model_classes = """	):
												lowerCAmelCase		=			len(snake_case__	) - len(line.lstrip()	)
												lowerCAmelCase		=			idx
								elif line.strip().startswith("""all_generative_model_classes = """	):
												lowerCAmelCase		=			len(snake_case__	) - len(line.lstrip()	)
												lowerCAmelCase		=			idx
								elif line.strip().startswith("""pipeline_model_mapping = """	):
												lowerCAmelCase		=			len(snake_case__	) - len(line.lstrip()	)
												lowerCAmelCase		=			idx
												lowerCAmelCase		=			line
												break

				if start_idx is None:
								return "", -1
				# Find the ending index (inclusive) of the above found block.
				lowerCAmelCase		=			find_block_ending(snake_case__ ,    snake_case__ ,    snake_case__	)

				# Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
				# `is_timm_available()` instead of `is_torch_available()`.
				# Keep leading and trailing whitespaces
				lowerCAmelCase		=			re.compile(R"""\s(is_\S+?_available\(\))\s"""	)
				for line in class_lines[start_idx : end_idx + 1]:
								lowerCAmelCase		=			r.search(snake_case__	)
								if backend_condition is not None:
												# replace the leading and trailing whitespaces to the space character " ".
												lowerCAmelCase		=			""" """ + backend_condition[0][1:-1] + """ """
												lowerCAmelCase		=			r.sub(snake_case__ ,    snake_case__	)
												break

				if def_line is None:
								# `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
								# `all_model_classes` or `all_generative_model_classes`.
								lowerCAmelCase		=			end_idx
				else:
								# `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
								lowerCAmelCase		=			start_idx - 1
								# mark the lines of the currently existing `pipeline_model_mapping` to be removed.
								for idx in range(snake_case__ ,    end_idx + 1	):
												# These lines are going to be removed before writing to the test file.
												lowerCAmelCase		=			None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
				lowerCAmelCase		=			[x.__name__ for x in test_class.__bases__]
				if "PipelineTesterMixin" not in parent_classes:
								# Put `PipelineTesterMixin` just before `unittest.TestCase`
								lowerCAmelCase		=			[x for x in parent_classes if x != """TestCase"""] + ["""PipelineTesterMixin"""]
								if "TestCase" in parent_classes:
												# Here we **assume** the original string is always with `unittest.TestCase`.
												_parent_classes.append("""unittest.TestCase"""	)
								lowerCAmelCase		=			""", """.join(_parent_classes	)
								for idx, line in enumerate(snake_case__	):
												# Find the ending of the declaration of `test_class`
												if line.strip().endswith("""):"""	):
																# mark the lines of the declaration of `test_class` to be removed
																for _idx in range(idx + 1	):
																				lowerCAmelCase		=			None  # noqa
																break
        # Add the new, one-line, class declaration for `test_class`
								lowerCAmelCase		=			f'''class {test_class.__name__}({parent_classes}):\n'''

				# Add indentation
				lowerCAmelCase		=			""" """ * indent_level + line_to_add
				# Insert `pipeline_model_mapping` to `class_lines`.
				# (The line at `target_idx` should be kept by definition!)
				lowerCAmelCase		=			class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
				# Remove the lines that are marked to be removed
				lowerCAmelCase		=			[x for x in class_lines if x is not None]

				# Move from test class to module (in order to write to the test file)
				lowerCAmelCase		=			inspect.getsourcelines(inspect.getmodule(snake_case__	)	)[0]
				# Be careful with the 1-off between line numbers and array indices
				lowerCAmelCase		=			module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
				lowerCAmelCase		=			"""""".join(snake_case__	)

				lowerCAmelCase		=			inspect.getsourcefile(snake_case__	)
				with open(snake_case__ ,    """w""" ,    encoding="""UTF-8""" ,    newline="""\n"""	) as fp:
								fp.write(snake_case__	)

				return line_to_add





def 							lowercase   (snake_case__				:	Union[str, Any] ,    snake_case__				:	List[Any]=False	)       ->					List[str]:


				'''simple docstring'''






				lowerCAmelCase		=			find_test_class(snake_case__	)
				if test_class:
								add_pipeline_model_mapping(snake_case__ ,    overwrite=snake_case__	)


if __name__ == "__main__":
		a								=							argparse.ArgumentParser()
		parser.add_argument(
		    '--test_file', type=str, help='A path to the test file, starting with the repository\'s `tests` directory.'
		)
		parser.add_argument(
		    '--all',
		    action='store_true',
		    help='If to check and modify all test files.',
		)
		parser.add_argument(
		    '--overwrite',
		    action='store_true',
		    help='If to overwrite a test class if it has already defined `pipeline_model_mapping`.',
		)
		a								=							parser.parse_args()

		if not args.all and not args.test_file:
				raise ValueError('Please specify either `test_file` or pass `--all` to check/modify all test files.')
		elif args.all and args.test_file:
				raise ValueError('Only one of `--test_file` and `--all` could be specified.')

		a								=							[]
		if args.test_file:
				a								=							[args.test_file]
		else:
				a								=							os.path.join('tests', 'models', '**', 'test_modeling_*.py')
				for test_file in glob.glob(pattern):
						# `Flax` is not concerned at this moment
						if not test_file.startswith('test_modeling_flax_'):
								test_files.append(test_file)

		for test_file in test_files:
				if test_file in TEST_FILE_TO_IGNORE:
						print(f"""[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.""")
						continue
				add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)











"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


a								=							logging.get_logger(__name__)

a								=							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a								=							{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

a								=							{
    'allenai/longformer-base-4096': 4_0_9_6,
    'allenai/longformer-large-4096': 4_0_9_6,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_0_9_6,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_0_9_6,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_0_9_6,
}






class  SCREAMING_SNAKE_CASE__   (   _a   ):
						_a        =  VOCAB_FILES_NAMES
						_a        =  PRETRAINED_VOCAB_FILES_MAP
						_a        =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a        =  ['input_ids', 'attention_mask']
						_a        =  LongformerTokenizer


						def __init__(							self			:     Optional[Any]  ,  lowerCAmelCase			:     Tuple=None  ,  lowerCAmelCase			:     Any=None  ,  lowerCAmelCase			:     List[Any]=None  ,  lowerCAmelCase			:     Optional[Any]="replace"  ,  lowerCAmelCase			:     Union[str, Any]="<s>"  ,  lowerCAmelCase			:     int="</s>"  ,  lowerCAmelCase			:     Any="</s>"  ,  lowerCAmelCase			:     Union[str, Any]="<s>"  ,  lowerCAmelCase			:     List[str]="<unk>"  ,  lowerCAmelCase			:     Optional[Any]="<pad>"  ,  lowerCAmelCase			:     List[str]="<mask>"  ,  lowerCAmelCase			:     Tuple=False  ,  lowerCAmelCase			:     List[str]=True  ,  **lowerCAmelCase			:     Optional[int]  ,  ):
										super().__init__(
										    lowerCAmelCase  ,  lowerCAmelCase  ,  tokenizer_file=lowerCAmelCase  ,  errors=lowerCAmelCase  ,  bos_token=lowerCAmelCase  ,  eos_token=lowerCAmelCase  ,  sep_token=lowerCAmelCase  ,  cls_token=lowerCAmelCase  ,  unk_token=lowerCAmelCase  ,  pad_token=lowerCAmelCase  ,  mask_token=lowerCAmelCase  ,  add_prefix_space=lowerCAmelCase  ,  trim_offsets=lowerCAmelCase  ,  **lowerCAmelCase  ,  )

										lowerCAmelCase		=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
										if pre_tok_state.get("""add_prefix_space"""  ,  lowerCAmelCase   ) != add_prefix_space:
														lowerCAmelCase		=			getattr(lowerCAmelCase  ,  pre_tok_state.pop("""type"""   )   )
														lowerCAmelCase		=			add_prefix_space
														lowerCAmelCase		=			pre_tok_class(**lowerCAmelCase   )

										lowerCAmelCase		=			add_prefix_space

										lowerCAmelCase		=			"""post_processor"""
										lowerCAmelCase		=			getattr(self.backend_tokenizer  ,  lowerCAmelCase  ,  lowerCAmelCase   )
										if tokenizer_component_instance:
														lowerCAmelCase		=			json.loads(tokenizer_component_instance.__getstate__()   )

														# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
														if "sep" in state:
																		lowerCAmelCase		=			tuple(state["""sep"""]   )
														if "cls" in state:
																		lowerCAmelCase		=			tuple(state["""cls"""]   )

														lowerCAmelCase		=			False

														if state.get("""add_prefix_space"""  ,  lowerCAmelCase   ) != add_prefix_space:
																		lowerCAmelCase		=			add_prefix_space
																		lowerCAmelCase		=			True

														if state.get("""trim_offsets"""  ,  lowerCAmelCase   ) != trim_offsets:
																		lowerCAmelCase		=			trim_offsets
																		lowerCAmelCase		=			True

														if changes_to_apply:
																		lowerCAmelCase		=			getattr(lowerCAmelCase  ,  state.pop("""type"""   )   )
																		lowerCAmelCase		=			component_class(**lowerCAmelCase   )
																		setattr(self.backend_tokenizer  ,  lowerCAmelCase  ,  lowerCAmelCase   )


						@property
						def 					__lowercase	(							self			:     List[Any]   ):
										if self._mask_token is None:
														if self.verbose:
																		logger.error("""Using mask_token, but it is not set yet."""   )
														return None
										return str(self._mask_token   )


						@mask_token.setter
						def 					__lowercase	(							self			:     List[Any]  ,  lowerCAmelCase			:     Dict   ):
										lowerCAmelCase		=			AddedToken(lowerCAmelCase  ,  lstrip=lowerCAmelCase  ,  rstrip=lowerCAmelCase   ) if isinstance(lowerCAmelCase  ,  lowerCAmelCase   ) else value
										lowerCAmelCase		=			value


						def 					__lowercase	(							self			:     Tuple  ,  *lowerCAmelCase			:     Optional[int]  ,  **lowerCAmelCase			:     Optional[Any]   ):
										lowerCAmelCase		=			kwargs.get("""is_split_into_words"""  ,  lowerCAmelCase   )
										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase  ,  **lowerCAmelCase   )


						def 					__lowercase	(							self			:     str  ,  *lowerCAmelCase			:     str  ,  **lowerCAmelCase			:     str   ):
										lowerCAmelCase		=			kwargs.get("""is_split_into_words"""  ,  lowerCAmelCase   )

										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase  ,  **lowerCAmelCase   )


						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     str  ,  lowerCAmelCase			:     Optional[str] = None   ):
										lowerCAmelCase		=			self._tokenizer.model.save(lowerCAmelCase  ,  name=lowerCAmelCase   )
										return tuple(lowerCAmelCase   )


						def 					__lowercase	(							self			:     Optional[Any]  ,  lowerCAmelCase			:     List[str]  ,  lowerCAmelCase			:     Optional[int]=None   ):
										lowerCAmelCase		=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
														return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




						def 					__lowercase	(							self			:     Optional[int]  ,  lowerCAmelCase			:     List[int]  ,  lowerCAmelCase			:     Optional[List[int]] = None   ):
										lowerCAmelCase		=			[self.sep_token_id]
										lowerCAmelCase		=			[self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]





