




# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


a_										=       logging.get_logger(__name__)


if is_torch_available():
 import torch
 from torch import nn
 from torch.utils.data import Dataset

 from transformers import Trainer

 class 	_lowerCamelCase  (     _SCREAMING_SNAKE_CASE				):

       """simple docstring"""

       def __init__(       self   :	Tuple	,    snake_case   :	int = 101			):
           __UpperCamelCase											=  length







       def __len__(       self   :	Tuple			):
           return self.length







       def __getitem__(       self   :	str	,    snake_case   :	Optional[int]			):
           return i







 class 	_lowerCamelCase  :

       """simple docstring"""

       def __call__(       self   :	Union[str, Any]	,    snake_case   :	str			):
           return {"input_ids": torch.tensor(snake_case			), "labels": torch.tensor(snake_case			)}







 class 	_lowerCamelCase  (     nn.Module				):

       """simple docstring"""

       def __init__(       self   :	int			):
           super().__init__()
           # Add some (unused) params otherwise DDP will complain.
           __UpperCamelCase											=  nn.Linear(120	,    80			)







       def  snake_case		(       self   :	str	,    snake_case   :	List[Any]	,    snake_case   :	Optional[int]=None			):
           if labels is not None:
               return torch.tensor(0.0	,    device=input_ids.device			), input_ids
           else:
               return input_ids




def    __SCREAMING_SNAKE_CASE				(					)    ->		List[str]:

    """simple docstring"""


    __UpperCamelCase											=  HfArgumentParser((TrainingArguments,) )
    sys.argv += ["--output_dir", "./examples"]
    __UpperCamelCase											=  parser.parse_args_into_dataclasses()[0]

    logger.warning(
        F"Process rank: {training_args.local_rank}, device: {training_args.device}, "
        F"tpu_num_cores: {training_args.tpu_num_cores}"   ,       )

    # Essentially, what we want to verify in the distributed case is
    # that we get all samples back, in the right order.
    # (this is crucial for prediction for instance)
    for dataset_length in [10_01, 2_56, 15]:
        __UpperCamelCase											=  DummyDataset(lowercase_ )

        def compute_metrics(lowercase_ ) -> Dict:
            __UpperCamelCase											=  list(range(len(lowercase_ ) ) )
            __UpperCamelCase											=  p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
            return {"success": success}

        __UpperCamelCase											=  Trainer(
            model=DummyModel()   ,       args=lowercase_   ,       data_collator=DummyDataCollator()   ,       eval_dataset=lowercase_   ,       compute_metrics=lowercase_   ,       )
        __UpperCamelCase											=  trainer.evaluate()
        logger.info(lowercase_ )
        if metrics["eval_success"] is not True:
            logger.error(lowercase_ )
            exit(1 )

        __UpperCamelCase											=  trainer.predict(lowercase_ )
        logger.info(p.metrics )
        if p.metrics["test_success"] is not True:
            logger.error(p.metrics )
            exit(1 )

        __UpperCamelCase											=  2

        __UpperCamelCase											=  trainer.evaluate()
        logger.info(lowercase_ )
        if metrics["eval_success"] is not True:
            logger.error(lowercase_ )
            exit(1 )

        __UpperCamelCase											=  trainer.predict(lowercase_ )
        logger.info(p.metrics )
        if p.metrics["test_success"] is not True:
            logger.error(p.metrics )
            exit(1 )

        __UpperCamelCase											=  None

    logger.info('''🔥 All distributed tests successful''' )




def    __SCREAMING_SNAKE_CASE				(					lowercase_ )    ->		Dict:

    """simple docstring"""


    main()


if __name__ == "__main__":
 main()












import argparse
import json
import re
import string
import sys
from collections import Counter




def    __SCREAMING_SNAKE_CASE				(					lowercase_ )    ->		Dict:

    """simple docstring"""


    def remove_articles(lowercase_ ):
        return re.sub(r'''\b(a|an|the)\b'''   ,       ''' '''   ,       lowercase_ )

    def white_space_fix(lowercase_ ):
        return " ".join(text.split() )

    def remove_punc(lowercase_ ):
        __UpperCamelCase											=  set(string.punctuation )
        return "".join(ch for ch in text if ch not in exclude )

    def lower(lowercase_ ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(lowercase_ ) ) ) )




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_ )    ->		List[str]:

    """simple docstring"""


    __UpperCamelCase											=  normalize_answer(lowercase_ ).split()
    __UpperCamelCase											=  normalize_answer(lowercase_ ).split()
    __UpperCamelCase											=  Counter(lowercase_ ) & Counter(lowercase_ )
    __UpperCamelCase											=  sum(common.values() )
    if num_same == 0:
        return 0
    __UpperCamelCase											=  1.0 * num_same / len(lowercase_ )
    __UpperCamelCase											=  1.0 * num_same / len(lowercase_ )
    __UpperCamelCase											=  (2 * precision * recall) / (precision + recall)
    return fa




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_ )    ->		Any:

    """simple docstring"""


    return normalize_answer(lowercase_ ) == normalize_answer(lowercase_ )




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_   ,       lowercase_ )    ->		int:

    """simple docstring"""


    __UpperCamelCase											=  []
    for ground_truth in ground_truths:
        __UpperCamelCase											=  metric_fn(lowercase_   ,       lowercase_ )
        scores_for_ground_truths.append(lowercase_ )
    return max(lowercase_ )




def    __SCREAMING_SNAKE_CASE				(					lowercase_   ,       lowercase_ )    ->		Optional[int]:

    """simple docstring"""


    __UpperCamelCase											=  __UpperCamelCase											=  __UpperCamelCase											=  0
    __UpperCamelCase											=  []
    for passage in dataset:
        for qa in passage["qas"]:
            total += 1
            if qa["id"] not in predictions:
                __UpperCamelCase											=  F"Unanswered question {qa['id']} will receive score 0."
                print(lowercase_   ,       file=sys.stderr )
                continue

            __UpperCamelCase											=  [x['''text'''] for x in qa['''answers''']]
            __UpperCamelCase											=  predictions[qa['''id''']]

            __UpperCamelCase											=  metric_max_over_ground_truths(lowercase_   ,       lowercase_   ,       lowercase_ )
            if int(_exact_match ) == 1:
                correct_ids.append(qa['''id'''] )
            exact_match += _exact_match

            fa += metric_max_over_ground_truths(lowercase_   ,       lowercase_   ,       lowercase_ )

    __UpperCamelCase											=  exact_match / total
    __UpperCamelCase											=  fa / total

    return {"exact_match": exact_match, "f1": fa}, correct_ids


if __name__ == "__main__":
 a_										=       "1.0"
 a_										=       argparse.ArgumentParser("Official evaluation script for ReCoRD v1.0.")
 parser.add_argument("data_file", help="The dataset file in JSON format.")
 parser.add_argument("pred_file", help="The model prediction file in JSON format.")
 parser.add_argument("--output_correct_ids", action="store_true", help="Output the correctly answered query IDs.")
 a_										=       parser.parse_args()

 with open(args.data_file) as data_file:
  a_										=       json.load(data_file)
  if dataset_json["version"] != expected_version:
   print(
       f"""Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json['version']}""",
       file=sys.stderr,
   )
  a_										=       dataset_json["data"]

 with open(args.pred_file) as pred_file:
  a_										=       json.load(pred_file)

 a_						,      a_										=       evaluate(dataset, predictions)

 if args.output_correct_ids:
  print(f"""Output {len(correct_ids)} correctly answered question IDs.""")
  with open("correct_ids.json", "w") as f:
   json.dump(correct_ids, f)






