
"""simple docstring"""




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


snake_case			       =		logging.getLogger(__name__)

def   UpperCamelCase_  (		):
					SCREAMING_SNAKE_CASE           =     argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'      )
					parser.add_argument(
					    '--dataset_name',					type=SCREAMING_SNAKE_CASE_,					default='wikitext',					help='Name of the training. Explore datasets at: hf.co/datasets.',					)
					parser.add_argument(
					    '--dataset_config',					type=SCREAMING_SNAKE_CASE_,					default='wikitext-103-raw-v1',					help='Configuration name of the dataset.'      )
					parser.add_argument(
					    '--batch_size',					type=SCREAMING_SNAKE_CASE_,					default=1_0_0_0,					help='Batch size during training.',					)
					parser.add_argument(
					    '--vocab_size',					type=SCREAMING_SNAKE_CASE_,					default=1_0_0_4_8,					help='Size of the desired vocabulary.',					)
					parser.add_argument(
					    '--limit',					default=SCREAMING_SNAKE_CASE_,					type=SCREAMING_SNAKE_CASE_,					help='Limit the number of shards (used for debugging).',					)
					parser.add_argument(
					    '--export_to_hub',					action='store_true',					)

					SCREAMING_SNAKE_CASE           =     parser.parse_args()
					return args

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     datasets.load_dataset(args.dataset_name,					args.dataset_config,					split='train'      )

					if args.limit is not None:
										SCREAMING_SNAKE_CASE           =     min(len(SCREAMING_SNAKE_CASE_      ),					args.limit      )
										SCREAMING_SNAKE_CASE           =     dataset.select(range(SCREAMING_SNAKE_CASE_      )      )
										logger.info(f'''Limiting the dataset to {args.limit} entries.'''      )

					def batch_iterator():
										for i in range(0,					len(SCREAMING_SNAKE_CASE_      ),					args.batch_size      ):
															yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
					SCREAMING_SNAKE_CASE           =     Tokenizer(Unigram()      )
					SCREAMING_SNAKE_CASE           =     normalizers.Sequence([normalizers.Replace('``',					'"'      ), normalizers.Replace('\'\'',					'"'      )]      )
					SCREAMING_SNAKE_CASE           =     pre_tokenizers.Metaspace()

					# Prepare the trainer.
					SCREAMING_SNAKE_CASE           =     UnigramTrainer(
					    unk_token='<unk>',					special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]'],					vocab_size=args.vocab_size,					)

					logger.info('Training the tokenizer.'      )
					tokenizer.train_from_iterator(batch_iterator(),					trainer=SCREAMING_SNAKE_CASE_      )
					logger.info('Tokenizer training complete!'      )

					SCREAMING_SNAKE_CASE           =     tokenizer.token_to_id('[CLS]'      )
					SCREAMING_SNAKE_CASE           =     tokenizer.token_to_id('[SEP]'      )
					SCREAMING_SNAKE_CASE           =     processors.TemplateProcessing(
					    single='[CLS]:0 $A:0 [SEP]:0',					pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1',					special_tokens=[
					        ('[CLS]', cls_token_id),
					        ('[SEP]', sep_token_id),
					    ],					)
					SCREAMING_SNAKE_CASE           =     decoders.Metaspace()

					if args.export_to_hub:
										logger.info('Exporting the trained tokenzier to Hub.'      )
										SCREAMING_SNAKE_CASE           =     AlbertTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE_      )
										new_tokenizer.push_to_hub('unigram-tokenizer-dataset'      )


if __name__ == "__main__":
	snake_case			       =		parse_args()
	main(args)


"""simple docstring"""




import argparse
import json
import re
import string
import sys
from collections import Counter

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):

					def remove_articles(SCREAMING_SNAKE_CASE_      ):
										return re.sub(R'\b(a|an|the)\b',					' ',					SCREAMING_SNAKE_CASE_      )

					def white_space_fix(SCREAMING_SNAKE_CASE_      ):
										return " ".join(text.split()      )

					def remove_punc(SCREAMING_SNAKE_CASE_      ):
										SCREAMING_SNAKE_CASE           =     set(string.punctuation      )
										return "".join(ch for ch in text if ch not in exclude      )

					def lower(SCREAMING_SNAKE_CASE_      ):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE_      )      )      )      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     normalize_answer(SCREAMING_SNAKE_CASE_      ).split()
					SCREAMING_SNAKE_CASE           =     normalize_answer(SCREAMING_SNAKE_CASE_      ).split()
					SCREAMING_SNAKE_CASE           =     Counter(SCREAMING_SNAKE_CASE_      ) & Counter(SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     sum(common.values()      )
					if num_same == 0:
										return 0
					SCREAMING_SNAKE_CASE           =     1.0 * num_same / len(SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     1.0 * num_same / len(SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     (2 * precision * recall) / (precision + recall)
					return fa

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					return normalize_answer(SCREAMING_SNAKE_CASE_      ) == normalize_answer(SCREAMING_SNAKE_CASE_      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     []
					for ground_truth in ground_truths:
										SCREAMING_SNAKE_CASE           =     metric_fn(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )
										scores_for_ground_truths.append(SCREAMING_SNAKE_CASE_      )
					return max(SCREAMING_SNAKE_CASE_      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     SCREAMING_SNAKE_CASE           =     SCREAMING_SNAKE_CASE           =     0
					for article in dataset:
										for paragraph in article["paragraphs"]:
															for qa in paragraph["qas"]:
																				total += 1
																				if qa["id"] not in predictions:
																									SCREAMING_SNAKE_CASE           =     'Unanswered question ' + qa['id'] + ' will receive score 0.'
																									print(SCREAMING_SNAKE_CASE_,					file=sys.stderr      )
																									continue
																				SCREAMING_SNAKE_CASE           =     [x['text'] for x in qa['answers']]
																				SCREAMING_SNAKE_CASE           =     predictions[qa['id']]
																				exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )
																				fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )

					SCREAMING_SNAKE_CASE           =     100.0 * exact_match / total
					SCREAMING_SNAKE_CASE           =     100.0 * fa / total

					return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
	snake_case			       =		'1.1'
	snake_case			       =		argparse.ArgumentParser(description='Evaluation for SQuAD ' + expected_version)
	parser.add_argument('dataset_file', help='Dataset file')
	parser.add_argument('prediction_file', help='Prediction File')
	snake_case			       =		parser.parse_args()
	with open(args.dataset_file) as dataset_file:
		snake_case			       =		json.load(dataset_file)
		if dataset_json["version"] != expected_version:
			print(
			    'Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'],
			    file=sys.stderr,
			)
		snake_case			       =		dataset_json['data']
	with open(args.prediction_file) as prediction_file:
		snake_case			       =		json.load(prediction_file)
	print(json.dumps(evaluate(dataset, predictions)))
