

'''simple docstring'''




import argparse
import logging
import os
import sys
import tempfile
from pathlib import Path

import lightning_base
import pytest
import pytorch_lightning as pl
import torch
from convert_pl_checkpoint_to_hf import convert_pl_to_hf
from distillation import distill_main
from finetune import SummarizationModule, main
from huggingface_hub import list_models
from parameterized import parameterized
from run_eval import generate_summaries_or_translations
from torch import nn

from transformers import AutoConfig, AutoModelForSeqaSeqLM
from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
from utils import label_smoothed_nll_loss, lmap, load_json


logging.basicConfig(level=logging.DEBUG)

_a			:	Optional[int]								=  logging.getLogger()
_a			:	str								=  torch.cuda.is_available()
_a			:	Optional[Any]								=  {
    'max_tokens_per_batch': None,
    'supervise_forward': True,
    'normalize_hidden': True,
    'label_smoothing': 0.2,
    'eval_max_gen_length': None,
    'eval_beams': 1,
    'val_metric': 'loss',
    'save_top_k': 1,
    'adafactor': True,
    'early_stopping_patience': 2,
    'logger_name': 'default',
    'length_penalty': 0.5,
    'cache_dir': '',
    'task': 'summarization',
    'num_workers': 2,
    'alpha_hid': 0,
    'freeze_embeds': True,
    'enc_only': False,
    'tgt_suffix': '',
    'resume_from_checkpoint': None,
    'sortish_sampler': True,
    'student_decoder_layers': 1,
    'val_check_interval': 1.0,
    'output_dir': '',
    'fp16': False,  # TODO(SS): set this to CUDA_AVAILABLE if ci installs apex or start using native amp
    'no_teacher': False,
    'fp16_opt_level': 'O1',
    'gpus': 1 if CUDA_AVAILABLE else 0,
    'n_tpu_cores': 0,
    'max_grad_norm': 1.0,
    'do_train': True,
    'do_predict': True,
    'accumulate_grad_batches': 1,
    'server_ip': '',
    'server_port': '',
    'seed': 42,
    'model_name_or_path': 'sshleifer/bart-tiny-random',
    'config_name': '',
    'tokenizer_name': 'facebook/bart-large',
    'do_lower_case': False,
    'learning_rate': 0.3,
    'lr_scheduler': 'linear',
    'weight_decay': 0.0,
    'adam_epsilon': 1E-08,
    'warmup_steps': 0,
    'max_epochs': 1,
    'train_batch_size': 2,
    'eval_batch_size': 2,
    'max_source_length': 12,
    'max_target_length': 12,
    'val_max_target_length': 12,
    'test_max_target_length': 12,
    'fast_dev_run': False,
    'no_cache': False,
    'n_train': -1,
    'n_val': -1,
    'n_test': -1,
    'student_encoder_layers': 1,
    'freeze_encoder': False,
    'auto_scale_batch_size': False,
    'overwrite_output_dir': False,
    'student': None,
}


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Path    ,				SCREAMING_SNAKE_CASE :				list  ):
		UpperCAmelCase  							=		'\n'.join(snake_case__  )
		Path(snake_case__  ).open('w'  ).writelines(snake_case__  )


_a			:	Tuple								=  [' Sam ate lunch today.', 'Sams lunch ingredients.']
_a			:	Any								=  ['A very interesting story about what I ate for lunch.', 'Avocado, celery, turkey, coffee']
_a			:	Optional[Any]								=  'patrickvonplaten/t5-tiny-random'
_a			:	Optional[Any]								=  'sshleifer/t5-tinier-random'
_a			:	Tuple								=  'sshleifer/bart-tiny-random'
_a			:	List[Any]								=  'sshleifer/tiny-mbart'
_a			:	Union[str, Any]								=  'sshleifer/tiny-marian-en-de'
_a			:	Dict								=  'stas/tiny-wmt19-en-de'


_a			:	Union[str, Any]								=  logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Union[str, Any]  ):






		for split in ["train", "val", "test"]:
				_dump_articles(os.path.join(snake_case__    ,				f'''{split}.source'''  )    ,				snake_case__  )
				_dump_articles(os.path.join(snake_case__    ,				f'''{split}.target'''  )    ,				snake_case__  )
		return tmp_dir



class     lowercase_		(						_a	):

		'''simple docstring'''





		@classmethod
		def   snake_case_     (       cls       )							->			Union[str, Any]:


				"""simple docstring"""
				logging.disable(logging.CRITICAL       )  # remove noisy download output from tracebacks
				return cls
		@slow
		@require_torch_gpu
		def   snake_case_     (       self       )							->			int:


				"""simple docstring"""
				UpperCAmelCase  							=		list_models()
				UpperCAmelCase  							=		'sshleifer'
				UpperCAmelCase  							=		[x.modelId for x in model_list if x.modelId.startswith(a_       )]
				UpperCAmelCase  							=		['sshleifer/blenderbot-3B', 'sshleifer/blenderbot-90M']
				UpperCAmelCase  							=		[]
				for m in model_ids:
						if m in allowed_to_be_broken:
								continue
						try:
								AutoConfig.from_pretrained(a_       )
						except Exception:
								failures.append(a_       )
				assert not failures, F'''The following models could not be loaded through AutoConfig: {failures}'''
		def   snake_case_     (       self       )							->			List[str]:


				"""simple docstring"""
				UpperCAmelCase  							=		{'student_encoder_layers': 2, 'student_decoder_layers': 1, 'no_teacher': True}
				self._test_distiller_cli(a_       )
		def   snake_case_     (       self       )							->			List[Any]:


				"""simple docstring"""
				UpperCAmelCase  							=		{
				    'student_encoder_layers': 2,
				    'student_decoder_layers': 1,
				    'max_epochs': 4,
				    'val_check_interval': 0.25,
				    'alpha_hid': 2.0,
				    'model_name_or_path': 'IGNORE_THIS_IT_DOESNT_GET_USED',
				}
				UpperCAmelCase  							=		self._test_distiller_cli(a_				,	check_contents=a_       )

				UpperCAmelCase  							=		list(Path(model.output_dir       ).glob('*.ckpt'       )       )
				self.assertEqual(1				,	len(a_       )       )
				UpperCAmelCase  							=		list(Path(model.output_dir       ).glob('**/*.bin'       )       )
				self.assertEqual(len(a_       )				,	2       )
				UpperCAmelCase  							=		lmap(str.strip				,	Path(model.hparams.data_dir       ).joinpath('test.source'       ).open().readlines()       )
				UpperCAmelCase  							=		tempfile.mktemp()  # XXX: not being cleaned up
				generate_summaries_or_translations(a_				,	a_				,	str(model.output_dir / 'best_tfmr'       )       )
				self.assertTrue(Path(a_       ).exists()       )

				UpperCAmelCase  							=		self.get_auto_remove_tmp_dir()
				convert_pl_to_hf(ckpts[0]				,	transformer_ckpts[0].parent				,	a_       )
				assert os.path.exists(os.path.join(a_				,	'pytorch_model.bin'       )       )
		def   snake_case_     (       self       )							->			str:


				"""simple docstring"""
				UpperCAmelCase  							=		AutoModelForSeqaSeqLM.from_pretrained(a_       )
				UpperCAmelCase							, UpperCAmelCase  							=		model.dummy_inputs['input_ids'], model.dummy_inputs['attention_mask']
				UpperCAmelCase  							=		torch.tensor([[0, 4, 8, 2], [0, 8, 2, 1]]				,	dtype=torch.long				,	device=model.device       )
				UpperCAmelCase  							=		target_ids[:, :-1].contiguous()  # Why this line?
				UpperCAmelCase  							=		target_ids[:, 1:].clone()  # why clone?
				UpperCAmelCase  							=		model(
				    a_				,	attention_mask=a_				,	decoder_input_ids=a_				,	labels=a_				,	use_cache=a_       ).loss

				UpperCAmelCase  							=		model(a_				,	attention_mask=a_				,	decoder_input_ids=a_				,	use_cache=a_       ).logits

				UpperCAmelCase  							=		nn.functional.log_softmax(a_				,	dim=-1       )
				UpperCAmelCase							, UpperCAmelCase  							=		label_smoothed_nll_loss(
				    a_				,	a_				,	0.1				,	ignore_index=model.config.pad_token_id       )
				with self.assertRaises(a_       ):
						# TODO: understand why this breaks
						self.assertEqual(a_				,	a_       )
		def   snake_case_     (       self       )							->			Tuple:


				"""simple docstring"""
				UpperCAmelCase  							=		{
				    'student_encoder_layers': 2,
				    'student_decoder_layers': 1,
				    'num_train_epochs': 4,
				    'val_check_interval': 0.25,
				    'alpha_hid': 2.0,
				    'task': 'translation',
				    'model_name_or_path': 'IGNORE_THIS_IT_DOESNT_GET_USED',
				    'tokenizer_name': MBART_TINY,
				    'teacher': MBART_TINY,
				    'src_lang': 'en_XX',
				    'tgt_lang': 'ro_RO',
				}
				UpperCAmelCase  							=		self._test_distiller_cli(a_				,	check_contents=a_       )
				assert model.model.config.model_type == "mbart"

				UpperCAmelCase  							=		list(Path(model.output_dir       ).glob('*.ckpt'       )       )
				self.assertEqual(1				,	len(a_       )       )
				UpperCAmelCase  							=		list(Path(model.output_dir       ).glob('**/*.bin'       )       )
				UpperCAmelCase  							=		list(Path(model.output_dir       ).glob('best_tfmr/*'       )       )
				assert len(a_       ) > 2
				self.assertEqual(len(a_       )				,	2       )
		def   snake_case_     (       self       )							->			Tuple:


				"""simple docstring"""
				UpperCAmelCase  							=		{
				    'student_encoder_layers': 1,
				    'student_decoder_layers': 1,
				    'alpha_hid': 2.0,
				    'teacher': T5_TINY,
				    'model_name_or_path': T5_TINY,
				    'tokenizer_name': T5_TINY,
				}
				self._test_distiller_cli(a_       )
		def   snake_case_     (       self       )							->			Optional[Any]:


				"""simple docstring"""
				UpperCAmelCase  							=		{
				    'teacher': T5_TINY,
				    'student': T5_TINIER,
				    'model_name_or_path': T5_TINIER,
				    'tokenizer_name': T5_TINIER,
				}
				self._test_distiller_cli(a_       )





		def   snake_case_     (       self				,	a_				,	a_=True       )							->			Optional[int]:


				"""simple docstring"""
				UpperCAmelCase  							=		{
				    'label_smoothing': 0.0,
				    'early_stopping_patience': -1,
				    'train_batch_size': 1,
				    'eval_batch_size': 2,
				    'max_epochs': 2,
				    'alpha_mlm': 0.2,
				    'alpha_ce': 0.8,
				    'do_predict': True,
				    'model_name_or_path': 'sshleifer/tinier_bart',
				    'teacher': CHEAP_ARGS['model_name_or_path'],
				    'val_check_interval': 0.5,
				}
				default_updates.update(a_       )
				UpperCAmelCase  							=		CHEAP_ARGS.copy()
				UpperCAmelCase  							=		make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()       )
				UpperCAmelCase  							=		self.get_auto_remove_tmp_dir()

				args_d.update(data_dir=a_				,	output_dir=a_				,	**a_       )
				UpperCAmelCase  							=		distill_main(argparse.Namespace(**a_       )       )
				if not check_contents:
						return model
				UpperCAmelCase  							=		os.listdir(a_       )
				UpperCAmelCase  							=		{os.path.basename(a_       ) for p in contents}
				UpperCAmelCase  							=		[p for p in contents if p.endswith('ckpt'       )]
				assert len(a_       ) > 0

				self.assertIn('test_generations.txt'				,	a_       )
				self.assertIn('test_results.txt'				,	a_       )

				UpperCAmelCase  							=		load_json(model.metrics_save_path       )
				UpperCAmelCase  							=		metrics['val'][-1]
				self.assertGreaterEqual(last_step_stats['val_avg_gen_time']				,	0.01       )
				self.assertGreaterEqual(1.0				,	last_step_stats['val_avg_gen_time']       )
				self.assertIsInstance(last_step_stats[F'''val_avg_{model.val_metric}''']				,	a_       )
				UpperCAmelCase  							=		int(args_d['max_epochs'] * (1 / args_d['val_check_interval']) + 1       )
				self.assertEqual(len(metrics['val']       )				,	a_       )
				self.assertEqual(len(metrics['test']       )				,	1       )
				return model




class     lowercase_		(						_a	):

		'''simple docstring'''





		@parameterized.expand(
		    [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY]				,	)
		def   snake_case_     (       self				,	a_       )							->			Dict:


				"""simple docstring"""
				UpperCAmelCase  							=		CHEAP_ARGS.copy()
				UpperCAmelCase  							=		'translation' if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else 'summarization'
				UpperCAmelCase  							=		0.1 if task == 'translation' else 0

				UpperCAmelCase  							=		make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()       )
				UpperCAmelCase  							=		self.get_auto_remove_tmp_dir()
				args_d.update(
				    data_dir=a_				,	model_name_or_path=a_				,	tokenizer_name=a_				,	train_batch_size=2				,	eval_batch_size=2				,	output_dir=a_				,	do_predict=a_				,	task=a_				,	src_lang='en_XX'				,	tgt_lang='ro_RO'				,	freeze_encoder=a_				,	freeze_embeds=a_				,	)
				assert "n_train" in args_d
				UpperCAmelCase  							=		argparse.Namespace(**a_       )
				UpperCAmelCase  							=		main(a_       )

				UpperCAmelCase  							=		module.model.get_input_embeddings()
				assert not input_embeds.weight.requires_grad
				if model == T5_TINY:
						UpperCAmelCase  							=		module.model.lm_head
						assert not lm_head.weight.requires_grad
						assert (lm_head.weight == input_embeds.weight).all().item()
				elif model == FSMT_TINY:
						UpperCAmelCase  							=		module.model.model
						UpperCAmelCase  							=		fsmt.decoder.embed_positions
						assert not embed_pos.weight.requires_grad
						assert not fsmt.decoder.embed_tokens.weight.requires_grad
						# check that embeds are not the same
						assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
				else:
						UpperCAmelCase  							=		module.model.model
						UpperCAmelCase  							=		bart.decoder.embed_positions
						assert not embed_pos.weight.requires_grad
						assert not bart.shared.weight.requires_grad
						# check that embeds are the same
						assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
						assert bart.decoder.embed_tokens == bart.shared

				UpperCAmelCase  							=		load_json(module.output_dir / 'text_batch.json'       )
				assert isinstance(a_				,	a_       )
				assert len(a_       ) >= 4
		def   snake_case_     (       self       )							->			Any:


				"""simple docstring"""
				UpperCAmelCase  							=		CHEAP_ARGS.copy()

				UpperCAmelCase  							=		'summarization'
				UpperCAmelCase  							=		make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()       )

				args_d.update(
				    data_dir=a_				,	tokenizer_name=a_				,	train_batch_size=2				,	eval_batch_size=2				,	do_predict=a_				,	task=a_				,	src_lang='en_XX'				,	tgt_lang='ro_RO'				,	freeze_encoder=a_				,	freeze_embeds=a_				,	)

				# test models whose config includes the extra_model_args
				UpperCAmelCase  							=		BART_TINY
				UpperCAmelCase  							=		self.get_auto_remove_tmp_dir()
				UpperCAmelCase  							=		args_d.copy()
				args_da.update(
				    model_name_or_path=a_				,	output_dir=a_				,	)
				UpperCAmelCase  							=		('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
				for p in extra_model_params:
						UpperCAmelCase  							=		0.5
				UpperCAmelCase  							=		argparse.Namespace(**a_       )
				UpperCAmelCase  							=		main(a_       )
				for p in extra_model_params:
						assert getattr(model.config				,	a_       ) == 0.5, F'''failed to override the model config for param {p}'''

				# test models whose config doesn't include the extra_model_args
				UpperCAmelCase  							=		T5_TINY
				UpperCAmelCase  							=		self.get_auto_remove_tmp_dir()
				UpperCAmelCase  							=		args_d.copy()
				args_da.update(
				    model_name_or_path=a_				,	output_dir=a_				,	)
				UpperCAmelCase  							=		'encoder_layerdrop'
				UpperCAmelCase  							=		0.5
				UpperCAmelCase  							=		argparse.Namespace(**a_       )
				with pytest.raises(a_       ) as excinfo:
						UpperCAmelCase  							=		main(a_       )
				assert str(excinfo.value       ) == F'''model config doesn\'t have a `{unsupported_param}` attribute'''





		def   snake_case_     (       self       )							->			str:


				"""simple docstring"""
				UpperCAmelCase  							=		CHEAP_ARGS.copy()

				UpperCAmelCase  							=		'summarization'
				UpperCAmelCase  							=		make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()       )

				UpperCAmelCase  							=		BART_TINY
				UpperCAmelCase  							=		self.get_auto_remove_tmp_dir()

				args_d.update(
				    data_dir=a_				,	model_name_or_path=a_				,	output_dir=a_				,	tokenizer_name=a_				,	train_batch_size=2				,	eval_batch_size=2				,	do_predict=a_				,	task=a_				,	src_lang='en_XX'				,	tgt_lang='ro_RO'				,	freeze_encoder=a_				,	freeze_embeds=a_				,	)

				# emulate finetune.py
				UpperCAmelCase  							=		argparse.ArgumentParser()
				UpperCAmelCase  							=		pl.Trainer.add_argparse_args(a_       )
				UpperCAmelCase  							=		SummarizationModule.add_model_specific_args(a_				,	os.getcwd()       )
				UpperCAmelCase  							=		{'--help': True}

				# --help test
				with pytest.raises(a_       ) as excinfo:
						with CaptureStdout() as cs:
								UpperCAmelCase  							=		parser.parse_args(a_       )
						assert False, "--help is expected to sys.exit"
				assert excinfo.type == SystemExit
				UpperCAmelCase  							=		lightning_base.arg_to_scheduler_metavar
				assert expected in cs.out, "--help is expected to list the supported schedulers"

				# --lr_scheduler=non_existing_scheduler test
				UpperCAmelCase  							=		'non_existing_scheduler'
				UpperCAmelCase  							=		{F'''--lr_scheduler={unsupported_param}'''}
				with pytest.raises(a_       ) as excinfo:
						with CaptureStderr() as cs:
								UpperCAmelCase  							=		parser.parse_args(a_       )
						assert False, "invalid argument is expected to sys.exit"
				assert excinfo.type == SystemExit
				UpperCAmelCase  							=		F'''invalid choice: \'{unsupported_param}\''''
				assert expected in cs.err, F'''should have bailed on invalid choice of scheduler {unsupported_param}'''

				# --lr_scheduler=existing_scheduler test
				UpperCAmelCase  							=		'cosine'
				UpperCAmelCase  							=		args_d.copy()
				UpperCAmelCase  							=		supported_param
				UpperCAmelCase  							=		argparse.Namespace(**a_       )
				UpperCAmelCase  							=		main(a_       )
				assert (
				    getattr(model.hparams				,	'lr_scheduler'       ) == supported_param
				), F'''lr_scheduler={supported_param} shouldn\'t fail'''





'''simple docstring'''




import argparse
import json
import re
import string
import sys
from collections import Counter


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				List[str]  ):

  def remove_articles(SCREAMING_SNAKE_CASE :				Union[str, Any]  ):
    return re.sub(R'\b(a|an|the)\b'    ,				' '    ,				SCREAMING_SNAKE_CASE  )

  def white_space_fix(SCREAMING_SNAKE_CASE :				Union[str, Any]  ):
    return " ".join(text.split()  )

  def remove_punc(SCREAMING_SNAKE_CASE :				List[Any]  ):
    UpperCAmelCase  							=		set(string.punctuation  )
    return "".join(ch for ch in text if ch not in exclude  )

  def lower(SCREAMING_SNAKE_CASE :				Any  ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE  )  )  )  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Tuple    ,				SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		normalize_answer(SCREAMING_SNAKE_CASE  ).split()
  UpperCAmelCase  							=		normalize_answer(SCREAMING_SNAKE_CASE  ).split()
  UpperCAmelCase  							=		Counter(SCREAMING_SNAKE_CASE  ) & Counter(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		sum(common.values()  )
  if num_same == 0:
    return 0
  UpperCAmelCase  							=		1.0 * num_same / len(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		1.0 * num_same / len(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		(2 * precision * recall) / (precision + recall)
  return fa


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				str    ,				SCREAMING_SNAKE_CASE :				int  ):
  return normalize_answer(SCREAMING_SNAKE_CASE  ) == normalize_answer(SCREAMING_SNAKE_CASE  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Dict    ,				SCREAMING_SNAKE_CASE :				Tuple    ,				SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		[]
  for ground_truth in ground_truths:
    UpperCAmelCase  							=		metric_fn(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )
    scores_for_ground_truths.append(SCREAMING_SNAKE_CASE  )
  return max(SCREAMING_SNAKE_CASE  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Optional[Any]    ,				SCREAMING_SNAKE_CASE :				Optional[int]  ):
  UpperCAmelCase  							=		UpperCAmelCase  							=		UpperCAmelCase  							=		0
  for article in dataset:
    for paragraph in article["paragraphs"]:
      for qa in paragraph["qas"]:
        total += 1
        if qa["id"] not in predictions:
          UpperCAmelCase  							=		'Unanswered question ' + qa['id'] + ' will receive score 0.'
          print(SCREAMING_SNAKE_CASE    ,				file=sys.stderr  )
          continue
        UpperCAmelCase  							=		[x['text'] for x in qa['answers']]
        UpperCAmelCase  							=		predictions[qa['id']]
        exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )
        fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )

  UpperCAmelCase  							=		100.0 * exact_match / total
  UpperCAmelCase  							=		100.0 * fa / total

  return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
  _a			:	Tuple								=  '1.1'
  _a			:	int								=  argparse.ArgumentParser(description='Evaluation for SQuAD ' + expected_version)
  parser.add_argument('dataset_file', help='Dataset file')
  parser.add_argument('prediction_file', help='Prediction File')
  _a			:	List[Any]								=  parser.parse_args()
  with open(args.dataset_file) as dataset_file:
    _a			:	Optional[int]								=  json.load(dataset_file)
    if dataset_json["version"] != expected_version:
      print(
          'Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'],
          file=sys.stderr,
      )
    _a			:	Optional[Any]								=  dataset_json['data']
  with open(args.prediction_file) as prediction_file:
    _a			:	Optional[Any]								=  json.load(prediction_file)
  print(json.dumps(evaluate(dataset, predictions)))


