

'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a			:	Optional[int]								=  logging.get_logger(__name__)


_a			:	List[str]								=  {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all MVP models at https://huggingface.co/models?filter=mvp
_a			:	Tuple								=  {
    'vocab_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json',
    },
    'added_tokens.json': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json',
    },
    'merges_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt',
    },
}

_a			:	str								=  {
    'RUCAIBox/mvp': 1_024,
}


@lru_cache()
def  lowerCamelCase__						(  ):
  UpperCAmelCase  							=		(
      list(range(ord('!'  )    ,				ord('~'  ) + 1  )  ) + list(range(ord('¡'  )    ,				ord('¬'  ) + 1  )  ) + list(range(ord('®'  )    ,				ord('ÿ'  ) + 1  )  )
  )
  UpperCAmelCase  							=		bs[:]
  UpperCAmelCase  							=		0
  for b in range(2**8  ):
    if b not in bs:
      bs.append(SCREAMING_SNAKE_CASE  )
      cs.append(2**8 + n  )
      n += 1
  UpperCAmelCase  							=		[chr(SCREAMING_SNAKE_CASE  ) for n in cs]
  return dict(zip(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				List[Any]  ):
  UpperCAmelCase  							=		set()
  UpperCAmelCase  							=		word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)  )
    UpperCAmelCase  							=		char
  return pairs



class     lowercase_		(						a	):

  '''simple docstring'''





  __lowerCAmelCase   :	str      					=				VOCAB_FILES_NAMES
  __lowerCAmelCase   :	List[Any]      					=				PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase   :	Dict      					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase   :	Optional[Any]      					=				["input_ids", "attention_mask"]
  def __init__(       self				,	a_				,	a_				,	a_="replace"				,	a_="<s>"				,	a_="</s>"				,	a_="</s>"				,	a_="<s>"				,	a_="<unk>"				,	a_="<pad>"				,	a_="<mask>"				,	a_=False				,	**a_				,	)							->			Optional[Any]:


    """simple docstring"""
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else bos_token
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else eos_token
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else sep_token
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else cls_token
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else unk_token
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    UpperCAmelCase  							=		AddedToken(a_				,	lstrip=a_				,	rstrip=a_       ) if isinstance(a_				,	a_       ) else mask_token

    super().__init__(
        errors=a_				,	bos_token=a_				,	eos_token=a_				,	unk_token=a_				,	sep_token=a_				,	cls_token=a_				,	pad_token=a_				,	mask_token=a_				,	add_prefix_space=a_				,	**a_				,	)
    with open(a_				,	encoding='utf-8'       ) as vocab_handle:
      UpperCAmelCase  							=		json.load(a_       )
    UpperCAmelCase  							=		{v: k for k, v in self.encoder.items()}
    UpperCAmelCase  							=		errors  # how to handle errors in decoding
    UpperCAmelCase  							=		bytes_to_unicode()
    UpperCAmelCase  							=		{v: k for k, v in self.byte_encoder.items()}
    with open(a_				,	encoding='utf-8'       ) as merges_handle:
      UpperCAmelCase  							=		merges_handle.read().split('\n'       )[1:-1]
    UpperCAmelCase  							=		[tuple(merge.split()       ) for merge in bpe_merges]
    UpperCAmelCase  							=		dict(zip(a_				,	range(len(a_       )       )       )       )
    UpperCAmelCase  							=		{}
    UpperCAmelCase  							=		add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    UpperCAmelCase  							=		re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'       )
  @property
  def   snake_case_     (       self       )							->			Any:


    """simple docstring"""
    return len(self.encoder       )
  def   snake_case_     (       self       )							->			Union[str, Any]:


    """simple docstring"""
    return dict(self.encoder				,	**self.added_tokens_encoder       )
  def   snake_case_     (       self				,	a_       )							->			int:


    """simple docstring"""
    if token in self.cache:
      return self.cache[token]
    UpperCAmelCase  							=		tuple(a_       )
    UpperCAmelCase  							=		get_pairs(a_       )

    if not pairs:
      return token

    while True:
      UpperCAmelCase  							=		min(a_				,	key=lambda a_       : self.bpe_ranks.get(a_				,	float('inf'       )       )       )
      if bigram not in self.bpe_ranks:
        break
      UpperCAmelCase							, UpperCAmelCase  							=		bigram
      UpperCAmelCase  							=		[]
      UpperCAmelCase  							=		0
      while i < len(a_       ):
        try:
          UpperCAmelCase  							=		word.index(a_				,	a_       )
        except ValueError:
          new_word.extend(word[i:]       )
          break
        else:
          new_word.extend(word[i:j]       )
          UpperCAmelCase  							=		j

        if word[i] == first and i < len(a_       ) - 1 and word[i + 1] == second:
          new_word.append(first + second       )
          i += 2
        else:
          new_word.append(word[i]       )
          i += 1
      UpperCAmelCase  							=		tuple(a_       )
      UpperCAmelCase  							=		new_word
      if len(a_       ) == 1:
        break
      else:
        UpperCAmelCase  							=		get_pairs(a_       )
    UpperCAmelCase  							=		' '.join(a_       )
    UpperCAmelCase  							=		word
    return word
  def   snake_case_     (       self				,	a_       )							->			int:


    """simple docstring"""
    UpperCAmelCase  							=		[]
    for token in re.findall(self.pat				,	a_       ):
      UpperCAmelCase  							=		''.join(
          self.byte_encoder[b] for b in token.encode('utf-8'       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
      bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_       ).split(' '       )       )
    return bpe_tokens
  def   snake_case_     (       self				,	a_       )							->			Optional[int]:


    """simple docstring"""
    return self.encoder.get(a_				,	self.encoder.get(self.unk_token       )       )
  def   snake_case_     (       self				,	a_       )							->			Tuple:


    """simple docstring"""
    return self.decoder.get(a_       )
  def   snake_case_     (       self				,	a_       )							->			List[Any]:


    """simple docstring"""
    UpperCAmelCase  							=		''.join(a_       )
    UpperCAmelCase  							=		bytearray([self.byte_decoder[c] for c in text]       ).decode('utf-8'				,	errors=self.errors       )
    return text
  def   snake_case_     (       self				,	a_				,	a_ = None       )							->			Tuple[str]:


    """simple docstring"""
    if not os.path.isdir(a_       ):
      logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
      return
    UpperCAmelCase  							=		os.path.join(
        a_				,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
    UpperCAmelCase  							=		os.path.join(
        a_				,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']       )

    with open(a_				,	'w'				,	encoding='utf-8'       ) as f:
      f.write(json.dumps(self.encoder				,	indent=2				,	sort_keys=a_				,	ensure_ascii=a_       ) + '\n'       )

    UpperCAmelCase  							=		0
    with open(a_				,	'w'				,	encoding='utf-8'       ) as writer:
      writer.write('#version: 0.2\n'       )
      for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,	key=lambda a_       : kv[1]       ):
        if index != token_index:
          logger.warning(
              F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              ' Please check that the tokenizer is not corrupted!'       )
          UpperCAmelCase  							=		token_index
        writer.write(' '.join(a_       ) + '\n'       )
        index += 1

    return vocab_file, merge_file
  def   snake_case_     (       self				,	a_				,	a_ = None       )							->			List[int]:


    """simple docstring"""
    if token_ids_a is None:
      return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    UpperCAmelCase  							=		[self.cls_token_id]
    UpperCAmelCase  							=		[self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep
  def   snake_case_     (       self				,	a_				,	a_ = None				,	a_ = False       )							->			List[int]:


    """simple docstring"""
    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=a_				,	token_ids_a=a_				,	already_has_special_tokens=a_       )

    if token_ids_a is None:
      return [1] + ([0] * len(a_       )) + [1]
    return [1] + ([0] * len(a_       )) + [1, 1] + ([0] * len(a_       )) + [1]
  def   snake_case_     (       self				,	a_				,	a_ = None       )							->			List[int]:


    """simple docstring"""
    UpperCAmelCase  							=		[self.sep_token_id]
    UpperCAmelCase  							=		[self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep       ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]





  def   snake_case_     (       self				,	a_				,	a_=False				,	**a_       )							->			Optional[Any]:


    """simple docstring"""
    UpperCAmelCase  							=		kwargs.pop('add_prefix_space'				,	self.add_prefix_space       )
    if (is_split_into_words or add_prefix_space) and (len(a_       ) > 0 and not text[0].isspace()):
      UpperCAmelCase  							=		' ' + text
    return (text, kwargs)





'''simple docstring'''




import argparse
import json
import re
import string
import sys
from collections import Counter


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				List[str]  ):

  def remove_articles(SCREAMING_SNAKE_CASE :				Union[str, Any]  ):
    return re.sub(R'\b(a|an|the)\b'    ,				' '    ,				SCREAMING_SNAKE_CASE  )

  def white_space_fix(SCREAMING_SNAKE_CASE :				Union[str, Any]  ):
    return " ".join(text.split()  )

  def remove_punc(SCREAMING_SNAKE_CASE :				List[Any]  ):
    UpperCAmelCase  							=		set(string.punctuation  )
    return "".join(ch for ch in text if ch not in exclude  )

  def lower(SCREAMING_SNAKE_CASE :				Any  ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE  )  )  )  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Tuple    ,				SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		normalize_answer(SCREAMING_SNAKE_CASE  ).split()
  UpperCAmelCase  							=		normalize_answer(SCREAMING_SNAKE_CASE  ).split()
  UpperCAmelCase  							=		Counter(SCREAMING_SNAKE_CASE  ) & Counter(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		sum(common.values()  )
  if num_same == 0:
    return 0
  UpperCAmelCase  							=		1.0 * num_same / len(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		1.0 * num_same / len(SCREAMING_SNAKE_CASE  )
  UpperCAmelCase  							=		(2 * precision * recall) / (precision + recall)
  return fa


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				str    ,				SCREAMING_SNAKE_CASE :				int  ):
  return normalize_answer(SCREAMING_SNAKE_CASE  ) == normalize_answer(SCREAMING_SNAKE_CASE  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Dict    ,				SCREAMING_SNAKE_CASE :				Tuple    ,				SCREAMING_SNAKE_CASE :				Dict  ):
  UpperCAmelCase  							=		[]
  for ground_truth in ground_truths:
    UpperCAmelCase  							=		metric_fn(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )
    scores_for_ground_truths.append(SCREAMING_SNAKE_CASE  )
  return max(SCREAMING_SNAKE_CASE  )


def  lowerCamelCase__						(  SCREAMING_SNAKE_CASE :				Optional[Any]    ,				SCREAMING_SNAKE_CASE :				Optional[int]  ):
  UpperCAmelCase  							=		UpperCAmelCase  							=		UpperCAmelCase  							=		0
  for article in dataset:
    for paragraph in article["paragraphs"]:
      for qa in paragraph["qas"]:
        total += 1
        if qa["id"] not in predictions:
          UpperCAmelCase  							=		'Unanswered question ' + qa['id'] + ' will receive score 0.'
          print(SCREAMING_SNAKE_CASE    ,				file=sys.stderr  )
          continue
        UpperCAmelCase  							=		[x['text'] for x in qa['answers']]
        UpperCAmelCase  							=		predictions[qa['id']]
        exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )
        fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE  )

  UpperCAmelCase  							=		100.0 * exact_match / total
  UpperCAmelCase  							=		100.0 * fa / total

  return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
  _a			:	Tuple								=  '1.1'
  _a			:	int								=  argparse.ArgumentParser(description='Evaluation for SQuAD ' + expected_version)
  parser.add_argument('dataset_file', help='Dataset file')
  parser.add_argument('prediction_file', help='Prediction File')
  _a			:	List[Any]								=  parser.parse_args()
  with open(args.dataset_file) as dataset_file:
    _a			:	Optional[int]								=  json.load(dataset_file)
    if dataset_json["version"] != expected_version:
      print(
          'Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'],
          file=sys.stderr,
      )
    _a			:	Optional[Any]								=  dataset_json['data']
  with open(args.prediction_file) as prediction_file:
    _a			:	Optional[Any]								=  json.load(prediction_file)
  print(json.dumps(evaluate(dataset, predictions)))


