

'''simple docstring'''




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_a			:	int								=  logging.getLogger(__name__)


def  lowerCamelCase__						(  ):
  UpperCAmelCase  							=		argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'  )
  parser.add_argument(
      '--dataset_name'    ,				type=SCREAMING_SNAKE_CASE    ,				default='wikitext'    ,				help='Name of the training. Explore datasets at: hf.co/datasets.'    ,				)
  parser.add_argument(
      '--dataset_config'    ,				type=SCREAMING_SNAKE_CASE    ,				default='wikitext-103-raw-v1'    ,				help='Configuration name of the dataset.'  )
  parser.add_argument(
      '--batch_size'    ,				type=SCREAMING_SNAKE_CASE    ,				default=1000    ,				help='Batch size during training.'    ,				)
  parser.add_argument(
      '--vocab_size'    ,				type=SCREAMING_SNAKE_CASE    ,				default=1_0048    ,				help='Size of the desired vocabulary.'    ,				)
  parser.add_argument(
      '--limit'    ,				default=SCREAMING_SNAKE_CASE    ,				type=SCREAMING_SNAKE_CASE    ,				help='Limit the number of shards (used for debugging).'    ,				)
  parser.add_argument(
      '--export_to_hub'    ,				action='store_true'    ,				)

  UpperCAmelCase  							=		parser.parse_args()
  return args


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		datasets.load_dataset(args.dataset_name    ,				args.dataset_config    ,				split='train'  )

  if args.limit is not None:
    UpperCAmelCase  							=		min(len(SCREAMING_SNAKE_CASE  )    ,				args.limit  )
    UpperCAmelCase  							=		dataset.select(range(SCREAMING_SNAKE_CASE  )  )
    logger.info(f'''Limiting the dataset to {args.limit} entries.'''  )

  def batch_iterator():
    for i in range(0    ,				len(SCREAMING_SNAKE_CASE  )    ,				args.batch_size  ):
      yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
  UpperCAmelCase  							=		Tokenizer(Unigram()  )
  UpperCAmelCase  							=		normalizers.Sequence([normalizers.Replace('``'    ,				'"'  ), normalizers.Replace('\'\''    ,				'"'  )]  )
  UpperCAmelCase  							=		pre_tokenizers.Metaspace()

  # Prepare the trainer.
  UpperCAmelCase  							=		UnigramTrainer(
      unk_token='<unk>'    ,				special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']    ,				vocab_size=args.vocab_size    ,				)

  logger.info('Training the tokenizer.'  )
  tokenizer.train_from_iterator(batch_iterator()    ,				trainer=SCREAMING_SNAKE_CASE  )
  logger.info('Tokenizer training complete!'  )

  UpperCAmelCase  							=		tokenizer.token_to_id('[CLS]'  )
  UpperCAmelCase  							=		tokenizer.token_to_id('[SEP]'  )
  UpperCAmelCase  							=		processors.TemplateProcessing(
      single='[CLS]:0 $A:0 [SEP]:0'    ,				pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'    ,				special_tokens=[
          ('[CLS]', cls_token_id),
          ('[SEP]', sep_token_id),
      ]    ,				)
  UpperCAmelCase  							=		decoders.Metaspace()

  if args.export_to_hub:
    logger.info('Exporting the trained tokenzier to Hub.'  )
    UpperCAmelCase  							=		AlbertTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE  )
    new_tokenizer.push_to_hub('unigram-tokenizer-dataset'  )


if __name__ == "__main__":
  _a			:	Any								=  parse_args()
  main(args)





'''simple docstring'''




import argparse
import json
import re
import string
import sys
from collections import Counter


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				List[str]  ):

  def remove_articles(SCREAMING_SNAKE_CASE :				Union[str, Any]  ):
    return re.sub(R'\b(a|an|the)\b'    ,				' '    ,				SCREAMING_SNAKE_CASE  )

  def white_space_fix(SCREAMING_SNAKE_CASE :				Union[str, Any]  ):
    return " ".join(text.split()  )

  def remove_punc(SCREAMING_SNAKE_CASE :				List[Any]  ):
    UpperCAmelCase  							=		set(string.punctuation  )
    return "".join(ch for ch in text if ch not in exclude  )

  def lower(SCREAMING_SNAKE_CASE :				Any  ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE  )  )  )  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Tuple    ,				SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		normalize_answer(SCREAMING_SNAKE_CASE  ).split()
  UpperCAmelCase  							=		normalize_answer(SCREAMING_SNAKE_CASE  ).split()
  UpperCAmelCase  							=		Counter(SCREAMING_SNAKE_CASE  ) & Counter(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		sum(common.values()  )
  if num_same == 0:
    return 0
  UpperCAmelCase  							=		1.0 * num_same / len(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		1.0 * num_same / len(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		(2 * precision * recall) / (precision + recall)
  return fa


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				str    ,				SCREAMING_SNAKE_CASE :				int  ):
  return normalize_answer(SCREAMING_SNAKE_CASE  ) == normalize_answer(SCREAMING_SNAKE_CASE  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Dict    ,				SCREAMING_SNAKE_CASE :				Tuple    ,				SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		[]
  for ground_truth in ground_truths:
    UpperCAmelCase  							=		metric_fn(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )
    scores_for_ground_truths.append(SCREAMING_SNAKE_CASE  )
  return max(SCREAMING_SNAKE_CASE  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Optional[Any]    ,				SCREAMING_SNAKE_CASE :				Optional[int]  ):
  UpperCAmelCase  							=		UpperCAmelCase  							=		UpperCAmelCase  							=		0
  for article in dataset:
    for paragraph in article["paragraphs"]:
      for qa in paragraph["qas"]:
        total += 1
        if qa["id"] not in predictions:
          UpperCAmelCase  							=		'Unanswered question ' + qa['id'] + ' will receive score 0.'
          print(SCREAMING_SNAKE_CASE    ,				file=sys.stderr  )
          continue
        UpperCAmelCase  							=		[x['text'] for x in qa['answers']]
        UpperCAmelCase  							=		predictions[qa['id']]
        exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )
        fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )

  UpperCAmelCase  							=		100.0 * exact_match / total
  UpperCAmelCase  							=		100.0 * fa / total

  return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
  _a			:	Tuple								=  '1.1'
  _a			:	int								=  argparse.ArgumentParser(description='Evaluation for SQuAD ' + expected_version)
  parser.add_argument('dataset_file', help='Dataset file')
  parser.add_argument('prediction_file', help='Prediction File')
  _a			:	List[Any]								=  parser.parse_args()
  with open(args.dataset_file) as dataset_file:
    _a			:	Optional[int]								=  json.load(dataset_file)
    if dataset_json["version"] != expected_version:
      print(
          'Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'],
          file=sys.stderr,
      )
    _a			:	Optional[Any]								=  dataset_json['data']
  with open(args.prediction_file) as prediction_file:
    _a			:	Optional[Any]								=  json.load(prediction_file)
  print(json.dumps(evaluate(dataset, predictions)))


