





"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE__ =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__ =       {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

SCREAMING_SNAKE_CASE__ =       {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

SCREAMING_SNAKE_CASE__ =       {
    "gpt2": 1_024,
    "gpt2-medium": 1_024,
    "gpt2-large": 1_024,
    "gpt2-xl": 1_024,
    "distilgpt2": 1_024,
}


@lru_cache()
def       UpperCAmelCase__							(  ):







      '''simple docstring'''




      lowerCAmelCase  						=	(
          list(range(ord("""!"""      )							,						ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )							,						ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )							,						ord("""ÿ"""      ) + 1      )      )
      )
      lowerCAmelCase  						=	bs[:]
      lowerCAmelCase  						=	0
      for b in range(2**8      ):
            if b not in bs:
                  bs.append(SCREAMING_SNAKE_CASE      )
                  cs.append(2**8 + n      )
                  n += 1
      lowerCAmelCase  						=	[chr(SCREAMING_SNAKE_CASE      ) for n in cs]
      return dict(zip(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE      )      )


def       UpperCAmelCase__							(  SCREAMING_SNAKE_CASE					:						Optional[int]      ):







      '''simple docstring'''




      lowerCAmelCase  						=	set()
      lowerCAmelCase  						=	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)      )
            lowerCAmelCase  						=	char
      return pairs






class 		lowercase ( SCREAMING_SNAKE_CASE_ ):
      _SCREAMING_SNAKE_CASE         =				VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE         =				PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE         =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE         =				['input_ids', 'attention_mask']





      def __init__(	self				,						lowercase				,						lowercase				,						lowercase="replace"				,						lowercase="<|endoftext|>"				,						lowercase="<|endoftext|>"				,						lowercase="<|endoftext|>"				,						lowercase=None				,						lowercase=False				,						lowercase=False				,						**lowercase				,						)		->					List[Any]:
            lowerCAmelCase  						=	AddedToken(SCREAMING_SNAKE_CASE_				,						lstrip=SCREAMING_SNAKE_CASE_				,						rstrip=SCREAMING_SNAKE_CASE_			) if isinstance(SCREAMING_SNAKE_CASE_				,						SCREAMING_SNAKE_CASE_			) else bos_token
            lowerCAmelCase  						=	AddedToken(SCREAMING_SNAKE_CASE_				,						lstrip=SCREAMING_SNAKE_CASE_				,						rstrip=SCREAMING_SNAKE_CASE_			) if isinstance(SCREAMING_SNAKE_CASE_				,						SCREAMING_SNAKE_CASE_			) else eos_token
            lowerCAmelCase  						=	AddedToken(SCREAMING_SNAKE_CASE_				,						lstrip=SCREAMING_SNAKE_CASE_				,						rstrip=SCREAMING_SNAKE_CASE_			) if isinstance(SCREAMING_SNAKE_CASE_				,						SCREAMING_SNAKE_CASE_			) else unk_token
            lowerCAmelCase  						=	AddedToken(SCREAMING_SNAKE_CASE_				,						lstrip=SCREAMING_SNAKE_CASE_				,						rstrip=SCREAMING_SNAKE_CASE_			) if isinstance(SCREAMING_SNAKE_CASE_				,						SCREAMING_SNAKE_CASE_			) else pad_token
            super().__init__(
                errors=SCREAMING_SNAKE_CASE_				,						unk_token=SCREAMING_SNAKE_CASE_				,						bos_token=SCREAMING_SNAKE_CASE_				,						eos_token=SCREAMING_SNAKE_CASE_				,						pad_token=SCREAMING_SNAKE_CASE_				,						add_prefix_space=SCREAMING_SNAKE_CASE_				,						add_bos_token=SCREAMING_SNAKE_CASE_				,						**SCREAMING_SNAKE_CASE_				,						)
            lowerCAmelCase  						=	add_bos_token

            with open(SCREAMING_SNAKE_CASE_				,						encoding="""utf-8"""			) as vocab_handle:
                  lowerCAmelCase  						=	json.load(SCREAMING_SNAKE_CASE_			)
            lowerCAmelCase  						=	{v: k for k, v in self.encoder.items()}
            lowerCAmelCase  						=	errors  # how to handle errors in decoding
            lowerCAmelCase  						=	bytes_to_unicode()
            lowerCAmelCase  						=	{v: k for k, v in self.byte_encoder.items()}
            with open(SCREAMING_SNAKE_CASE_				,						encoding="""utf-8"""			) as merges_handle:
                  lowerCAmelCase  						=	merges_handle.read().split("""\n"""			)[1:-1]
            lowerCAmelCase  						=	[tuple(merge.split()			) for merge in bpe_merges]
            lowerCAmelCase  						=	dict(zip(SCREAMING_SNAKE_CASE_				,						range(len(SCREAMING_SNAKE_CASE_			)			)			)			)
            lowerCAmelCase  						=	{}
            lowerCAmelCase  						=	add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            lowerCAmelCase  						=	re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)





      @property
      def    _snake_case      (	self			)		->					Union[str, Any]:
            return len(self.encoder			)





      def    _snake_case      (	self			)		->					List[Any]:
            return dict(self.encoder				,						**self.added_tokens_encoder			)





      def    _snake_case      (	self				,						lowercase			)		->					Optional[Any]:






            if token in self.cache:
                  return self.cache[token]
            lowerCAmelCase  						=	tuple(SCREAMING_SNAKE_CASE_			)
            lowerCAmelCase  						=	get_pairs(SCREAMING_SNAKE_CASE_			)

            if not pairs:
                  return token

            while True:
                  lowerCAmelCase  						=	min(SCREAMING_SNAKE_CASE_				,						key=lambda lowercase			: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_				,						float("""inf"""			)			)			)
                  if bigram not in self.bpe_ranks:
                        break
                  lowerCAmelCase						,		lowerCAmelCase  						=	bigram
                  lowerCAmelCase  						=	[]
                  lowerCAmelCase  						=	0
                  while i < len(SCREAMING_SNAKE_CASE_			):
                        try:
                              lowerCAmelCase  						=	word.index(SCREAMING_SNAKE_CASE_				,						SCREAMING_SNAKE_CASE_			)
                        except ValueError:
                              new_word.extend(word[i:]			)
                              break
                        else:
                              new_word.extend(word[i:j]			)
                              lowerCAmelCase  						=	j

                        if word[i] == first and i < len(SCREAMING_SNAKE_CASE_			) - 1 and word[i + 1] == second:
                              new_word.append(first + second			)
                              i += 2
                        else:
                              new_word.append(word[i]			)
                              i += 1
                  lowerCAmelCase  						=	tuple(SCREAMING_SNAKE_CASE_			)
                  lowerCAmelCase  						=	new_word
                  if len(SCREAMING_SNAKE_CASE_			) == 1:
                        break
                  else:
                        lowerCAmelCase  						=	get_pairs(SCREAMING_SNAKE_CASE_			)
            lowerCAmelCase  						=	""" """.join(SCREAMING_SNAKE_CASE_			)
            lowerCAmelCase  						=	word
            return word





      def    _snake_case      (	self				,						lowercase				,						lowercase=None			)		->					int:






            if self.add_bos_token:
                  lowerCAmelCase  						=	[self.bos_token_id]
            else:
                  lowerCAmelCase  						=	[]

            lowerCAmelCase  						=	bos_token_ids + token_ids_a

            if token_ids_a is None:
                  return output

            return output + bos_token_ids + token_ids_a





      def    _snake_case      (	self				,						lowercase				,						lowercase = None				,						lowercase = False			)		->					List[int]:






            if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=SCREAMING_SNAKE_CASE_				,						token_ids_a=SCREAMING_SNAKE_CASE_				,						already_has_special_tokens=SCREAMING_SNAKE_CASE_			)

            if not self.add_bos_token:
                  return super().get_special_tokens_mask(
                      token_ids_a=SCREAMING_SNAKE_CASE_				,						token_ids_a=SCREAMING_SNAKE_CASE_				,						already_has_special_tokens=SCREAMING_SNAKE_CASE_			)

            if token_ids_a is None:
                  return [1] + ([0] * len(SCREAMING_SNAKE_CASE_			))
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE_			)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE_			))





      def    _snake_case      (	self				,						lowercase			)		->					int:
            lowerCAmelCase  						=	[]
            for token in re.findall(self.pat				,						SCREAMING_SNAKE_CASE_			):
                  lowerCAmelCase  						=	"""""".join(
                      self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                  bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE_			).split(""" """			)			)
            return bpe_tokens





      def    _snake_case      (	self				,						lowercase			)		->					Optional[Any]:
            return self.encoder.get(SCREAMING_SNAKE_CASE_				,						self.encoder.get(self.unk_token			)			)





      def    _snake_case      (	self				,						lowercase			)		->					int:
            return self.decoder.get(SCREAMING_SNAKE_CASE_			)





      def    _snake_case      (	self				,						lowercase			)		->					Dict:
            lowerCAmelCase  						=	"""""".join(SCREAMING_SNAKE_CASE_			)
            lowerCAmelCase  						=	bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8"""				,						errors=self.errors			)
            return text





      def    _snake_case      (	self				,						lowercase				,						lowercase = None			)		->					Tuple[str]:






            if not os.path.isdir(SCREAMING_SNAKE_CASE_			):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory'			)
                  return
            lowerCAmelCase  						=	os.path.join(
                SCREAMING_SNAKE_CASE_				,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
            lowerCAmelCase  						=	os.path.join(
                SCREAMING_SNAKE_CASE_				,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

            with open(SCREAMING_SNAKE_CASE_				,						"""w"""				,						encoding="""utf-8"""			) as f:
                  f.write(json.dumps(self.encoder				,						indent=2				,						sort_keys=SCREAMING_SNAKE_CASE_				,						ensure_ascii=SCREAMING_SNAKE_CASE_			) + """\n"""			)

            lowerCAmelCase  						=	0
            with open(SCREAMING_SNAKE_CASE_				,						"""w"""				,						encoding="""utf-8"""			) as writer:
                  writer.write("""#version: 0.2\n"""			)
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,						key=lambda lowercase			: kv[1]			):
                        if index != token_index:
                              logger.warning(
                                  f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                  """ Please check that the tokenizer is not corrupted!"""			)
                              lowerCAmelCase  						=	token_index
                        writer.write(""" """.join(SCREAMING_SNAKE_CASE_			) + """\n"""			)
                        index += 1

            return vocab_file, merge_file





      def    _snake_case      (	self				,						lowercase				,						lowercase=False				,						**lowercase			)		->					str:
            lowerCAmelCase  						=	kwargs.pop("""add_prefix_space"""				,						self.add_prefix_space			)
            if is_split_into_words or add_prefix_space:
                  lowerCAmelCase  						=	""" """ + text
            return (text, kwargs)





      def    _snake_case      (	self				,						lowercase			)		->					List[int]:
            lowerCAmelCase  						=	[]
            for is_user, text in conversation.iter_texts():
                  input_ids.extend(self.encode(SCREAMING_SNAKE_CASE_				,						add_special_tokens=SCREAMING_SNAKE_CASE_			) + [self.eos_token_id]			)
            if len(SCREAMING_SNAKE_CASE_			) > self.model_max_length:
                  lowerCAmelCase  						=	input_ids[-self.model_max_length :]
            return input_ids













"""simple docstring"""





import argparse
import json
import re
import string
import sys
from collections import Counter


def       UpperCAmelCase__							(  SCREAMING_SNAKE_CASE					:						List[Any]      ):







      '''simple docstring'''




      def remove_articles(SCREAMING_SNAKE_CASE					:						int      ):
            return re.sub(R"""\b(a|an|the)\b"""							,						""" """							,						SCREAMING_SNAKE_CASE      )

      def white_space_fix(SCREAMING_SNAKE_CASE					:						int      ):
            return " ".join(text.split()      )

      def remove_punc(SCREAMING_SNAKE_CASE					:						Union[str, Any]      ):
            lowerCAmelCase  						=	set(string.punctuation      )
            return "".join(ch for ch in text if ch not in exclude      )

      def lower(SCREAMING_SNAKE_CASE					:						Optional[Any]      ):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE      )      )      )      )


def       UpperCAmelCase__							(  SCREAMING_SNAKE_CASE					:						Union[str, Any]							,						SCREAMING_SNAKE_CASE					:						Optional[int]      ):







      '''simple docstring'''




      lowerCAmelCase  						=	normalize_answer(SCREAMING_SNAKE_CASE      ).split()
      lowerCAmelCase  						=	normalize_answer(SCREAMING_SNAKE_CASE      ).split()
      lowerCAmelCase  						=	Counter(SCREAMING_SNAKE_CASE      ) & Counter(SCREAMING_SNAKE_CASE      )
      lowerCAmelCase  						=	sum(common.values()      )
      if num_same == 0:
            return 0
      lowerCAmelCase  						=	1.0 * num_same / len(SCREAMING_SNAKE_CASE      )
      lowerCAmelCase  						=	1.0 * num_same / len(SCREAMING_SNAKE_CASE      )
      lowerCAmelCase  						=	(2 * precision * recall) / (precision + recall)
      return fa


def       UpperCAmelCase__							(  SCREAMING_SNAKE_CASE					:						Tuple							,						SCREAMING_SNAKE_CASE					:						Tuple      ):







      '''simple docstring'''




      return normalize_answer(SCREAMING_SNAKE_CASE      ) == normalize_answer(SCREAMING_SNAKE_CASE      )


def       UpperCAmelCase__							(  SCREAMING_SNAKE_CASE					:						List[Any]							,						SCREAMING_SNAKE_CASE					:						Any							,						SCREAMING_SNAKE_CASE					:						str      ):







      '''simple docstring'''




      lowerCAmelCase  						=	[]
      for ground_truth in ground_truths:
            lowerCAmelCase  						=	metric_fn(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE      )
            scores_for_ground_truths.append(SCREAMING_SNAKE_CASE      )
      return max(SCREAMING_SNAKE_CASE      )


def       UpperCAmelCase__							(  SCREAMING_SNAKE_CASE					:						Optional[int]							,						SCREAMING_SNAKE_CASE					:						Dict      ):







      '''simple docstring'''




      lowerCAmelCase  						=	lowerCAmelCase  						=	lowerCAmelCase  						=	0
      for article in dataset:
            for paragraph in article["paragraphs"]:
                  for qa in paragraph["qas"]:
                        total += 1
                        if qa["id"] not in predictions:
                              lowerCAmelCase  						=	"""Unanswered question """ + qa["""id"""] + """ will receive score 0."""
                              print(SCREAMING_SNAKE_CASE							,						file=sys.stderr      )
                              continue
                        lowerCAmelCase  						=	[x["""text"""] for x in qa["""answers"""]]
                        lowerCAmelCase  						=	predictions[qa["""id"""]]
                        exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE      )
                        fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE							,						SCREAMING_SNAKE_CASE      )

      lowerCAmelCase  						=	1_00.0 * exact_match / total
      lowerCAmelCase  						=	1_00.0 * fa / total

      return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
       SCREAMING_SNAKE_CASE__ =       "1.1"
       SCREAMING_SNAKE_CASE__ =       argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version)
       parser.add_argument("dataset_file", help="Dataset file")
       parser.add_argument("prediction_file", help="Prediction File")
       SCREAMING_SNAKE_CASE__ =       parser.parse_args()
       with open(args.dataset_file) as dataset_file:
              SCREAMING_SNAKE_CASE__ =       json.load(dataset_file)
              if dataset_json["version"] != expected_version:
                     print(
                         "Evaluation expects v-" + expected_version + ", but got dataset with v-" + dataset_json["version"],
                         file=sys.stderr,
                     )
              SCREAMING_SNAKE_CASE__ =       dataset_json["data"]
       with open(args.prediction_file) as prediction_file:
              SCREAMING_SNAKE_CASE__ =       json.load(prediction_file)
       print(json.dumps(evaluate(dataset, predictions)))






