






"""simple docstring"""



import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


__A		:	str      =		{0: '''null''', 1: '''short''', 2: '''long''', 3: '''yes''', 4: '''no'''}
__A		:	Union[str, Any]      =		set(''''''.join(['''‘''', '''’''', '''´''', '''`''', '''.''', ''',''', '''-''', '''"''']))



def 						lowercase   (    __snake_case    :			Tuple		, __snake_case    :			Optional[Any]=0		, __snake_case    :			List[str]=None     ):
		return [" ".join(x.split(''' '''     )[begin:end]     ) for x in answers if len(x.split(''' '''     )     ) > 1]



def 						lowercase   (    __snake_case    :			List[Any]		, __snake_case    :			Any=False     ):
		if make_sub_answers:
				# if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
				# *e.g.* if the correct answer contains a prefix such as "the", or "a"
				lowercase_			:			List[Any]								=       (
				    given_answers + get_sub_answers(__snake_case		, begin=1     ) + get_sub_answers(__snake_case		, end=-1     )
				)
		lowercase_			:			List[str]								=       []
		for answer in given_answers:
				lowercase_			:			str								=       answer.replace('''_'''		, ''' '''     ).lower()
				lowercase_			:			Dict								=       ''''''.join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else ''' ''' for c in alias     )
				answers.append(''' '''.join(alias.split()     ).strip()     )
		return set(__snake_case     )



def 						lowercase   (    __snake_case    :			Tuple		, __snake_case    :			Tuple		, __snake_case    :			Optional[int]=1		, __snake_case    :			List[Any]=1_0_0     ):
		lowercase_			:			str								=       jax.lax.top_k(__snake_case		, __snake_case     )
		lowercase_			:			int								=       jax.lax.top_k(__snake_case		, __snake_case     )

		lowercase_			:			Optional[Any]								=       best_end_idx[:, None] - best_start_idx[None, :]
		lowercase_			:			Optional[int]								=       jnp.logical_or(widths < 0		, widths > max_size     )
		lowercase_			:			List[Any]								=       (best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
		lowercase_			:			Optional[int]								=       jnp.argmax(__snake_case     ).item()

		return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]



def 						lowercase   (    __snake_case    :			Union[str, Any]     ):
		lowercase_			:			int								=       sample['''question''']['''text''']
		lowercase_			:			List[str]								=       sample['''document''']['''tokens''']['''token''']
		lowercase_			:			Dict								=       sample['''document''']['''tokens''']['''is_html''']
		lowercase_			:			Union[str, Any]								=       sample['''annotations''']['''long_answer''']
		lowercase_			:			Union[str, Any]								=       sample['''annotations''']['''short_answers''']

		lowercase_			:			Any								=       ''' '''.join([context[i] for i in range(len(__snake_case     )     ) if not is_html[i]]     )

		# 0 - No ; 1 - Yes
		for answer in sample["annotations"]["yes_no_answer"]:
				if answer == 0 or answer == 1:
						return {
						    "question": question,
						    "context": context_string,
						    "short": [],
						    "long": [],
						    "category": "no" if answer == 0 else "yes",
						}

		lowercase_			:			Optional[Any]								=       []
		for s in short_answers:
				short_targets.extend(s['''text''']     )
		lowercase_			:			str								=       list(set(__snake_case     )     )

		lowercase_			:			List[Any]								=       []
		for s in long_answers:
				if s["start_token"] == -1:
						continue
				lowercase_			:			List[Any]								=       context[s['''start_token'''] : s['''end_token''']]
				lowercase_			:			Dict								=       is_html[s['''start_token'''] : s['''end_token''']]
				lowercase_			:			Tuple								=       ''' '''.join([answer[i] for i in range(len(__snake_case     )     ) if not html[i]]     )
				if new_answer not in long_targets:
						long_targets.append(__snake_case     )

		lowercase_			:			int								=       '''long_short''' if len(short_targets + long_targets     ) > 0 else '''null'''

		return {
		    "question": question,
		    "context": context_string,
		    "short": short_targets,
		    "long": long_targets,
		    "category": category,
		}



def 						lowercase   (    ):
		lowercase_			:			str								=       load_from_disk('''natural-questions-validation'''     )
		lowercase_			:			Tuple								=       dataset.map(__snake_case     ).remove_columns(['''annotations''', '''document''', '''id''']     )
		print(__snake_case     )

		lowercase_			:			Optional[Any]								=       dataset.filter(lambda __snake_case     : (len(x['''question''']     ) + len(x['''context''']     )) < 4 * 4_0_9_6     )
		lowercase_			:			Union[str, Any]								=       short_validation_dataset.filter(lambda __snake_case     : x["category"] != "null"     )
		short_validation_dataset

		lowercase_			:			Tuple								=       '''vasudevgupta/flax-bigbird-natural-questions'''
		lowercase_			:			Dict								=       FlaxBigBirdForNaturalQuestions.from_pretrained(__snake_case     )
		lowercase_			:			Union[str, Any]								=       BigBirdTokenizerFast.from_pretrained(__snake_case     )

		@jax.jit
		def forward(*__snake_case    :			Any		, **__snake_case    :			Optional[int]     ):
				lowercase_			:			List[str]								=       model(*__snake_case		, **__snake_case     )
				return start_logits, end_logits, jnp.argmax(__snake_case		, axis=-1     )

		def evaluate(__snake_case    :			Dict     ):
				# encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
				lowercase_			:			int								=       tokenizer(
				    example['''question''']		, example['''context''']		, return_tensors='''np'''		, max_length=4_0_9_6		, padding='''max_length'''		, truncation=__snake_case		, )

				lowercase_			:			Any								=       forward(**__snake_case     )

				lowercase_			:			Optional[int]								=       CATEGORY_MAPPING[category.item()]

				lowercase_			:			Union[str, Any]								=       example['''long'''] + example['''short''']
				if example["category"] in ["yes", "no", "null"]:
						lowercase_			:			str								=       [example['''category''']]
				lowercase_			:			Any								=       example['''category'''] != '''null'''
				# Now target can be: "yes", "no", "null", "list of long & short answers"

				if predicted_category in ["yes", "no", "null"]:
						lowercase_			:			Dict								=       [predicted_category]
						lowercase_			:			Optional[int]								=       example['''output'''] == example['''targets''']
						lowercase_			:			Optional[int]								=       predicted_category != '''null'''
						return example

				lowercase_			:			Dict								=       3_8 if predicted_category == '''short''' else 1_0_2_4
				lowercase_			:			Any								=       get_best_valid_start_end_idx(
				    start_scores[0]		, end_scores[0]		, top_k=8		, max_size=__snake_case     )

				lowercase_			:			Tuple								=       inputs['''input_ids'''][0].tolist()
				lowercase_			:			str								=       [tokenizer.decode(input_ids[start_score : end_score + 1]     )]

				lowercase_			:			List[Any]								=       expand_to_aliases(example['''targets''']		, make_sub_answers=__snake_case     )
				lowercase_			:			Optional[int]								=       expand_to_aliases(example['''output''']     )

				# some preprocessing to both prediction and answer
				lowercase_			:			Union[str, Any]								=       {''''''.join(a.split()     ) for a in answers}
				lowercase_			:			Any								=       {''''''.join(p.split()     ) for p in predictions}
				lowercase_			:			int								=       {s for s in predictions if s not in ['''``''', '''\'\'''', '''`''', '''\'''']}

				# if there is a common element, it's a exact match
				lowercase_			:			List[Any]								=       len(list(answers & predictions     )     ) > 0
				lowercase_			:			List[str]								=       predicted_category != '''null''' and len(__snake_case     ) > 0

				return example

		lowercase_			:			Optional[Any]								=       short_validation_dataset.map(__snake_case     )

		lowercase_			:			Union[str, Any]								=       len(__snake_case     )
		lowercase_			:			Tuple								=       len(short_validation_dataset.filter(lambda __snake_case     : x["match"] == 1     )     )
		print('''EM score:'''		, (matched / total) * 1_0_0		, '''%'''     )


if __name__ == "__main__":
	main()











"""simple docstring"""



import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor



def 						lowercase   (    ):
		lowercase_			:			Tuple								=       set()
		# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
		for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
				lowercase_			:			str								=       getattr(transformers.models.auto		, __snake_case		, __snake_case     )
				if module is None:
						continue
				# all mappings in a single auto modeling file
				lowercase_			:			Tuple								=       [
				    x
				    for x in dir(__snake_case     )
				    if x.endswith('''_MAPPING_NAMES'''     )
				    and (x.startswith('''MODEL_'''     ) or x.startswith('''TF_MODEL_'''     ) or x.startswith('''FLAX_MODEL_'''     ))
				]
				for name in mapping_names:
						lowercase_			:			Tuple								=       getattr(__snake_case		, __snake_case     )
						if mapping is not None:
								for v in mapping.values():
										if isinstance(__snake_case		, (list, tuple)     ):
												model_names.update(__snake_case     )
										elif isinstance(__snake_case		, __snake_case     ):
												model_names.add(__snake_case     )

		return sorted(__snake_case     )



def 						lowercase   (    ):
		# All model names defined in auto mappings
		lowercase_			:			Tuple								=       set(get_all_model_names()     )

		with open('''tests/utils/tiny_model_summary.json'''     ) as fp:
				lowercase_			:			Tuple								=       json.load(__snake_case     )
		lowercase_			:			Union[str, Any]								=       set()
		for model_base_name in tiny_model_info:
				tiny_models_names.update(tiny_model_info[model_base_name]['''model_classes''']     )

		# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
		lowercase_			:			Union[str, Any]								=       model_names.difference(__snake_case     )
		for model_name in copy.copy(__snake_case     ):
				if not model_name.startswith('''TF'''     ) and F'''TF{model_name}''' in not_on_hub:
						tiny_models_names.remove(__snake_case     )
				elif model_name.startswith('''TF'''     ) and model_name[2:] in not_on_hub:
						tiny_models_names.remove(__snake_case     )

		return sorted(__snake_case     )



def 						lowercase   (    __snake_case    :			List[Any]     ):
		lowercase_			:			List[str]								=       COMPOSITE_MODELS.values()

		# All tiny model base names on Hub
		lowercase_			:			List[Any]								=       get_all_model_names()
		lowercase_			:			Tuple								=       hf_api.list_models(
		    filter=ModelFilter(
		        author='''hf-internal-testing'''		, )     )
		lowercase_			:			Tuple								=       set()
		for x in models:
				lowercase_			:			Tuple								=       x.modelId
				lowercase_	, lowercase_			:			Tuple								=       model.split('''/'''     )
				if not model.startswith('''tiny-random-'''     ):
						continue
				lowercase_			:			List[str]								=       model.replace('''tiny-random-'''		, ''''''     )
				if not model[0].isupper():
						continue
				if model not in model_names and model not in special_models:
						continue
				_models.add(__snake_case     )

		lowercase_			:			Tuple								=       sorted(_models     )
		# All tiny model names on Hub
		lowercase_			:			int								=       {}
		for model in models:
				lowercase_			:			str								=       F'''hf-internal-testing/tiny-random-{model}'''
				lowercase_			:			str								=       model.split('''-'''     )[0]
				try:
						lowercase_			:			Any								=       hf_api.repo_info(__snake_case     )
						lowercase_			:			str								=       {
						    '''tokenizer_classes''': set(),
						    '''processor_classes''': set(),
						    '''model_classes''': set(),
						    '''sha''': repo_info.sha,
						}
				except Exception:
						continue
				try:
						time.sleep(1     )
						lowercase_			:			Any								=       AutoTokenizer.from_pretrained(__snake_case     )
						content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__     )
				except Exception:
						pass
				try:
						time.sleep(1     )
						lowercase_			:			List[str]								=       AutoTokenizer.from_pretrained(__snake_case		, use_fast=__snake_case     )
						content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__     )
				except Exception:
						pass
				try:
						time.sleep(1     )
						lowercase_			:			Union[str, Any]								=       AutoImageProcessor.from_pretrained(__snake_case     )
						content["processor_classes"].add(img_p.__class__.__name__     )
				except Exception:
						pass
				try:
						time.sleep(1     )
						lowercase_			:			Optional[Any]								=       AutoFeatureExtractor.from_pretrained(__snake_case     )
						if not isinstance(__snake_case		, __snake_case     ):
								content["processor_classes"].add(feat_p.__class__.__name__     )
				except Exception:
						pass
				try:
						time.sleep(1     )
						lowercase_			:			Union[str, Any]								=       getattr(__snake_case		, __snake_case     )
						lowercase_			:			Dict								=       model_class.from_pretrained(__snake_case     )
						content["model_classes"].add(m.__class__.__name__     )
				except Exception:
						pass
				try:
						time.sleep(1     )
						lowercase_			:			List[str]								=       getattr(__snake_case		, F'''TF{model}'''     )
						lowercase_			:			List[str]								=       model_class.from_pretrained(__snake_case     )
						content["model_classes"].add(m.__class__.__name__     )
				except Exception:
						pass

				lowercase_			:			Dict								=       sorted(content['''tokenizer_classes''']     )
				lowercase_			:			Any								=       sorted(content['''processor_classes''']     )
				lowercase_			:			Union[str, Any]								=       sorted(content['''model_classes''']     )

				lowercase_			:			str								=       content
				with open(os.path.join(__snake_case		, '''hub_tiny_model_summary.json'''     )		, '''w'''     ) as fp:
						json.dump(__snake_case		, __snake_case		, ensure_ascii=__snake_case		, indent=4     )


if __name__ == "__main__":
						__A		:	Optional[int]      =		argparse.ArgumentParser()
						parser.add_argument('''--num_workers''', default=1, type=int, help='''The number of workers to run.''')
						__A		:	Optional[int]      =		parser.parse_args()

						# This has to be `spawn` to avoid hanging forever!
						multiprocessing.set_start_method('''spawn''')

						__A		:	List[str]      =		'''tiny_models'''
						__A		:	List[str]      =		True
						__A		:	int      =		None
						__A		:	Dict      =		get_tiny_model_names_from_repo()
						__A		:	Union[str, Any]      =		True
						__A		:	str      =		True
						__A		:	Optional[Any]      =		'''hf-internal-testing'''

						create_tiny_models(
						    output_path,
						    all,
						    model_types,
						    models_to_skip,
						    no_check,
						    upload,
						    organization,
						    token=os.environ.get('''TOKEN''', None),
						    num_workers=args.num_workers,
						)



