

'''simple docstring'''

import jax
import jax.numpy as jnp
from bigbird_flax import FlaxBigBirdForNaturalQuestions
from datasets import load_from_disk

from transformers import BigBirdTokenizerFast


a       :      str  	=			{0: "null", 1: "short", 2: "long", 3: "yes", 4: "no"}
a       :      str  	=			set("".join(["‘", "’", "´", "`", ".", ",", "-", "\""]))





def 				lowercase (					__magic_name__			,   __magic_name__=0			,   __magic_name__=None							):



					'''simple docstring'''







					return [" ".join(x.split(" "							)[begin:end]							) for x in answers if len(x.split(" "							)							) > 1]





def 				lowercase (					__magic_name__			,   __magic_name__=False							):



					'''simple docstring'''







					if make_sub_answers:
										# if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
										# *e.g.* if the correct answer contains a prefix such as "the", or "a"
										UpperCAmelCase		:							Any        =       (
										    given_answers + get_sub_answers(__magic_name__			,   begin=1							) + get_sub_answers(__magic_name__			,   end=-1							)
										)
					UpperCAmelCase		:							str        =       []
					for answer in given_answers:
										UpperCAmelCase		:							Optional[Any]        =       answer.replace("_"			,   " "							).lower()
										UpperCAmelCase		:							Optional[Any]        =       "".join(c if c not in PUNCTUATION_SET_TO_EXCLUDE else " " for c in alias							)
										answers.append(" ".join(alias.split()							).strip()							)
					return set(__magic_name__							)





def 				lowercase (					__magic_name__			,   __magic_name__			,   __magic_name__=1			,   __magic_name__=100							):



					'''simple docstring'''







					UpperCAmelCase ,			UpperCAmelCase		:							Any        =       jax.lax.top_k(__magic_name__			,   __magic_name__							)
					UpperCAmelCase ,			UpperCAmelCase		:							Any        =       jax.lax.top_k(__magic_name__			,   __magic_name__							)

					UpperCAmelCase		:							Optional[Any]        =       best_end_idx[:, None] - best_start_idx[None, :]
					UpperCAmelCase		:							List[Any]        =       jnp.logical_or(widths < 0			,   widths > max_size							)
					UpperCAmelCase		:							int        =       (best_end_scores[:, None] + best_start_scores[None, :]) - (1e8 * mask)
					UpperCAmelCase		:							Dict        =       jnp.argmax(__magic_name__							).item()

					return best_start_idx[best_score % top_k], best_end_idx[best_score // top_k]





def 				lowercase (					__magic_name__							):



					'''simple docstring'''







					UpperCAmelCase		:							Dict        =       sample["question"]["text"]
					UpperCAmelCase		:							Union[str, Any]        =       sample["document"]["tokens"]["token"]
					UpperCAmelCase		:							Optional[Any]        =       sample["document"]["tokens"]["is_html"]
					UpperCAmelCase		:							Tuple        =       sample["annotations"]["long_answer"]
					UpperCAmelCase		:							Union[str, Any]        =       sample["annotations"]["short_answers"]

					UpperCAmelCase		:							Tuple        =       " ".join([context[i] for i in range(len(__magic_name__							)							) if not is_html[i]]							)

					# 0 - No ; 1 - Yes
					for answer in sample["annotations"]["yes_no_answer"]:
										if answer == 0 or answer == 1:
															return {
															    "question": question,
															    "context": context_string,
															    "short": [],
															    "long": [],
															    "category": "no" if answer == 0 else "yes",
															}

					UpperCAmelCase		:							int        =       []
					for s in short_answers:
										short_targets.extend(s["text"]							)
					UpperCAmelCase		:							Dict        =       list(set(__magic_name__							)							)

					UpperCAmelCase		:							str        =       []
					for s in long_answers:
										if s["start_token"] == -1:
															continue
										UpperCAmelCase		:							Any        =       context[s["start_token"] : s["end_token"]]
										UpperCAmelCase		:							Any        =       is_html[s["start_token"] : s["end_token"]]
										UpperCAmelCase		:							Optional[int]        =       " ".join([answer[i] for i in range(len(__magic_name__							)							) if not html[i]]							)
										if new_answer not in long_targets:
															long_targets.append(__magic_name__							)

					UpperCAmelCase		:							str        =       "long_short" if len(short_targets + long_targets							) > 0 else "null"

					return {
					    "question": question,
					    "context": context_string,
					    "short": short_targets,
					    "long": long_targets,
					    "category": category,
					}





def 				lowercase (					):



					'''simple docstring'''







					UpperCAmelCase		:							int        =       load_from_disk("natural-questions-validation"							)
					UpperCAmelCase		:							str        =       dataset.map(__magic_name__							).remove_columns(["annotations", "document", "id"]							)
					print(__magic_name__							)

					UpperCAmelCase		:							List[Any]        =       dataset.filter(lambda __magic_name__							: (len(x["question"]							) + len(x["context"]							)) < 4 * 4096							)
					UpperCAmelCase		:							str        =       short_validation_dataset.filter(lambda __magic_name__							: x["category"] != "null"							)
					short_validation_dataset

					UpperCAmelCase		:							Optional[int]        =       "vasudevgupta/flax-bigbird-natural-questions"
					UpperCAmelCase		:							List[str]        =       FlaxBigBirdForNaturalQuestions.from_pretrained(__magic_name__							)
					UpperCAmelCase		:							Any        =       BigBirdTokenizerFast.from_pretrained(__magic_name__							)

					@jax.jit
					def forward(*__magic_name__			,   **__magic_name__							):
										UpperCAmelCase ,			UpperCAmelCase ,			UpperCAmelCase		:							Any        =       model(*__magic_name__			,   **__magic_name__							)
										return start_logits, end_logits, jnp.argmax(__magic_name__			,   axis=-1							)

					def evaluate(__magic_name__							):
										# encode question and context so that they are separated by a tokenizer.sep_token and cut at max_length
										UpperCAmelCase		:							Tuple        =       tokenizer(
										    example["question"]			,   example["context"]			,   return_tensors="np"			,   max_length=4096			,   padding="max_length"			,   truncation=__magic_name__			,   )

										UpperCAmelCase ,			UpperCAmelCase ,			UpperCAmelCase		:							Dict        =       forward(**__magic_name__							)

										UpperCAmelCase		:							str        =       CATEGORY_MAPPING[category.item()]

										UpperCAmelCase		:							str        =       example["long"] + example["short"]
										if example["category"] in ["yes", "no", "null"]:
															UpperCAmelCase		:							List[str]        =       [example["category"]]
										UpperCAmelCase		:							Optional[int]        =       example["category"] != "null"
										# Now target can be: "yes", "no", "null", "list of long & short answers"

										if predicted_category in ["yes", "no", "null"]:
															UpperCAmelCase		:							Tuple        =       [predicted_category]
															UpperCAmelCase		:							Union[str, Any]        =       example["output"] == example["targets"]
															UpperCAmelCase		:							Dict        =       predicted_category != "null"
															return example

										UpperCAmelCase		:							Any        =       38 if predicted_category == "short" else 1024
										UpperCAmelCase ,			UpperCAmelCase		:							str        =       get_best_valid_start_end_idx(
										    start_scores[0]			,   end_scores[0]			,   top_k=8			,   max_size=__magic_name__							)

										UpperCAmelCase		:							str        =       inputs["input_ids"][0].tolist()
										UpperCAmelCase		:							Optional[int]        =       [tokenizer.decode(input_ids[start_score : end_score + 1]							)]

										UpperCAmelCase		:							List[Any]        =       expand_to_aliases(example["targets"]			,   make_sub_answers=__magic_name__							)
										UpperCAmelCase		:							int        =       expand_to_aliases(example["output"]							)

										# some preprocessing to both prediction and answer
										UpperCAmelCase		:							str        =       {"".join(a.split()							) for a in answers}
										UpperCAmelCase		:							int        =       {"".join(p.split()							) for p in predictions}
										UpperCAmelCase		:							int        =       {s for s in predictions if s not in ["``", "''", "`", "'"]}

										# if there is a common element, it's a exact match
										UpperCAmelCase		:							Dict        =       len(list(answers & predictions							)							) > 0
										UpperCAmelCase		:							List[Any]        =       predicted_category != "null" and len(__magic_name__							) > 0

										return example

					UpperCAmelCase		:							List[Any]        =       short_validation_dataset.map(__magic_name__							)

					UpperCAmelCase		:							Optional[int]        =       len(__magic_name__							)
					UpperCAmelCase		:							List[Any]        =       len(short_validation_dataset.filter(lambda __magic_name__							: x["match"] == 1							)							)
					print("EM score:"			,   (matched / total) * 100			,   "%"							)


if __name__ == "__main__":
				main()



'''simple docstring'''

import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only

from utils import save_json





def 				lowercase (					__magic_name__							):



					'''simple docstring'''







					UpperCAmelCase		:							Any        =       filter(lambda __magic_name__							: p.requires_grad			,   model.parameters()							)
					UpperCAmelCase		:							Union[str, Any]        =       sum([np.prod(p.size()							) for p in model_parameters]							)
					return params


a       :      int  	=			logging.getLogger(__name__)







class   UpperCamelCase__   (       pl.Callback       ):


		"""simple docstring"""

		def 				A_	(			self   ,      snake_case   ,      snake_case      ):


							'''simple docstring'''

							UpperCAmelCase		:							Union[str, Any]        =       {f"lr_group_{i}": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups      )}
							pl_module.logger.log_metrics(snake_case      )







		@rank_zero_only
		def 				A_	(			self   ,      snake_case   ,      snake_case   ,      snake_case   ,      snake_case=True      ):


							'''simple docstring'''

							logger.info(f"***** {type_path} results at step {trainer.global_step:05d} *****"      )
							UpperCAmelCase		:							Optional[Any]        =       trainer.callback_metrics
							trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]}      )
							# Log results
							UpperCAmelCase		:							List[str]        =       Path(pl_module.hparams.output_dir      )
							if type_path == "test":
												UpperCAmelCase		:							Dict        =       od / "test_results.txt"
												UpperCAmelCase		:							Dict        =       od / "test_generations.txt"
							else:
												# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
												# If people want this it will be easy enough to add back.
												UpperCAmelCase		:							Union[str, Any]        =       od / f"{type_path}_results/{trainer.global_step:05d}.txt"
												UpperCAmelCase		:							int        =       od / f"{type_path}_generations/{trainer.global_step:05d}.txt"
												results_file.parent.mkdir(exist_ok=snake_case      )
												generations_file.parent.mkdir(exist_ok=snake_case      )
							with open(snake_case   ,      "a+"      ) as writer:
												for key in sorted(snake_case      ):
																	if key in ["log", "progress_bar", "preds"]:
																						continue
																	UpperCAmelCase		:							Optional[Any]        =       metrics[key]
																	if isinstance(snake_case   ,      torch.Tensor      ):
																						UpperCAmelCase		:							Union[str, Any]        =       val.item()
																	UpperCAmelCase		:							List[str]        =       f"{key}: {val:.6f}\n"
																	writer.write(snake_case      )

							if not save_generations:
												return

							if "preds" in metrics:
												UpperCAmelCase		:							List[str]        =       "\n".join(metrics["preds"]      )
												generations_file.open("w+"      ).write(snake_case      )







		@rank_zero_only
		def 				A_	(			self   ,      snake_case   ,      snake_case      ):


							'''simple docstring'''

							try:
												UpperCAmelCase		:							Tuple        =       pl_module.model.model.num_parameters()
							except AttributeError:
												UpperCAmelCase		:							int        =       pl_module.model.num_parameters()

							UpperCAmelCase		:							List[Any]        =       count_trainable_parameters(snake_case      )
							# mp stands for million parameters
							trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1e6, "grad_mp": n_trainable_pars / 1e6}      )







		@rank_zero_only
		def 				A_	(			self   ,      snake_case   ,      snake_case      ):


							'''simple docstring'''

							save_json(pl_module.metrics   ,      pl_module.metrics_save_path      )
							return self._write_logs(snake_case   ,      snake_case   ,      "test"      )







		@rank_zero_only
		def 				A_	(			self   ,      snake_case   ,      snake_case      ):


							'''simple docstring'''

							save_json(pl_module.metrics   ,      pl_module.metrics_save_path      )
							# Uncommenting this will save val generations
							# return self._write_logs(trainer, pl_module, "valid")





def 				lowercase (					__magic_name__			,   __magic_name__			,   __magic_name__=1			,   __magic_name__=False							):



					'''simple docstring'''







					if metric == "rouge2":
										UpperCAmelCase		:							Dict        =       "{val_avg_rouge2:.4f}-{step_count}"
					elif metric == "bleu":
										UpperCAmelCase		:							int        =       "{val_avg_bleu:.4f}-{step_count}"
					elif metric == "loss":
										UpperCAmelCase		:							Tuple        =       "{val_avg_loss:.4f}-{step_count}"
					else:
										raise NotImplementedError(
										    F"seq2seq callbacks only support rouge2, bleu and loss, got {metric}, You can make your own by adding to"
										    " this function."							)

					UpperCAmelCase		:							int        =       ModelCheckpoint(
					    dirpath=__magic_name__			,   filename=__magic_name__			,   monitor=F"val_{metric}"			,   mode="min" if "loss" in metric else "max"			,   save_top_k=__magic_name__			,   )
					return checkpoint_callback





def 				lowercase (					__magic_name__			,   __magic_name__							):



					'''simple docstring'''







					return EarlyStopping(
					    monitor=F"val_{metric}"			,   mode="min" if "loss" in metric else "max"			,   patience=__magic_name__			,   verbose=__magic_name__			,   )
