




"""simple docstring"""





import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata

def 						lowerCAmelCase							(	lowerCamelCase						):
						'''simple docstring'''


						snake_case__    :   Dict							     =   min(re.search('''^ +''' ,     lowerCamelCase						).end() if t.startswith(''' '''						) else 0 for t in string.splitlines()						)
						return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(lowerCamelCase						)]						)


_lowercase		 =      """\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
"""

_lowercase		 =      """\
---
---
# Begin of markdown

Some cool dataset card
"""


_lowercase		 =      """\
# Begin of markdown

Some cool dataset card
"""

class  __lowercase    (     unittest.TestCase		):


				'''simple docstring'''


				def  _lowerCamelCase   (				self				:       int ):
										with tempfile.TemporaryDirectory() as tmp_dir:
																snake_case__    :   int							     =   Path(SCREAMING_SNAKE_CASE_ ) / '''README.md'''
																with open(SCREAMING_SNAKE_CASE_ ,'''w+''' ) as readme_file:
																						readme_file.write(SCREAMING_SNAKE_CASE_ )
																snake_case__    :   List[str]							     =   DatasetMetadata.from_readme(SCREAMING_SNAKE_CASE_ )
																self.assertDictEqual(SCREAMING_SNAKE_CASE_ ,{'''language''': ['''zh''', '''en'''], '''task_ids''': ['''sentiment-classification''']} )

																with open(SCREAMING_SNAKE_CASE_ ,'''w+''' ) as readme_file:
																						readme_file.write(SCREAMING_SNAKE_CASE_ )
																snake_case__    :   Tuple							     =   DatasetMetadata.from_readme(SCREAMING_SNAKE_CASE_ )
																self.assertDictEqual(SCREAMING_SNAKE_CASE_ ,{} )

																with open(SCREAMING_SNAKE_CASE_ ,'''w+''' ) as readme_file:
																						readme_file.write(SCREAMING_SNAKE_CASE_ )
																snake_case__    :   List[Any]							     =   DatasetMetadata.from_readme(SCREAMING_SNAKE_CASE_ )
																self.assertEqual(SCREAMING_SNAKE_CASE_ ,{} )





				def  _lowerCamelCase   (				self				:       List[Any] ):
										snake_case__    :   str							     =   _dedent(
										    '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            ''' )
										assert DatasetMetadata.from_yaml_string(SCREAMING_SNAKE_CASE_ )

										snake_case__    :   Union[str, Any]							     =   _dedent(
										    '''\
            annotations_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            task_ids:
            - open-domain-qa
            ''' )
										with self.assertRaises(SCREAMING_SNAKE_CASE_ ):
																DatasetMetadata.from_yaml_string(SCREAMING_SNAKE_CASE_ )

										snake_case__    :   int							     =   _dedent(
										    '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - text-classification
            task_ids:
            - multi-class-classification
            paperswithcode_id:
            - squad
            configs:
            - en
            train-eval-index:
            - config: en
              task: text-classification
              task_id: multi_class_classification
              splits:
                train_split: train
                eval_split: test
              col_mapping:
                text: text
                label: target
              metrics:
                - type: accuracy
                  name: Accuracy
            extra_gated_prompt: |
              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:
              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:
              1. Researcher shall use the Database only for non-commercial research and educational purposes.
            extra_gated_fields:
              Company: text
              Country: text
              I agree to use this model for non-commerical use ONLY: checkbox
            ''' )
										assert DatasetMetadata.from_yaml_string(SCREAMING_SNAKE_CASE_ )








"""simple docstring"""





import argparse
import json
import re
import string
import sys
from collections import Counter

def 						lowerCAmelCase							(	lowerCamelCase						):
						'''simple docstring'''


						def remove_articles(lowerCamelCase						):
												return re.sub(R'''\b(a|an|the)\b''' ,     ''' ''' ,     lowerCamelCase						)

						def white_space_fix(lowerCamelCase						):
												return " ".join(text.split()						)

						def remove_punc(lowerCamelCase						):
												snake_case__    :   Tuple							     =   set(string.punctuation						)
												return "".join(ch for ch in text if ch not in exclude						)

						def lower(lowerCamelCase						):
												return text.lower()

						return white_space_fix(remove_articles(remove_punc(lower(lowerCamelCase						)						)						)						)

def 						lowerCAmelCase							(	lowerCamelCase ,     lowerCamelCase						):
						'''simple docstring'''


						snake_case__    :   Dict							     =   normalize_answer(lowerCamelCase						).split()
						snake_case__    :   Dict							     =   normalize_answer(lowerCamelCase						).split()
						snake_case__    :   List[Any]							     =   Counter(lowerCamelCase						) & Counter(lowerCamelCase						)
						snake_case__    :   List[str]							     =   sum(common.values()						)
						if num_same == 0:
												return 0
						snake_case__    :   int							     =   1.0 * num_same / len(lowerCamelCase						)
						snake_case__    :   str							     =   1.0 * num_same / len(lowerCamelCase						)
						snake_case__    :   Union[str, Any]							     =   (2 * precision * recall) / (precision + recall)
						return fa

def 						lowerCAmelCase							(	lowerCamelCase ,     lowerCamelCase						):
						'''simple docstring'''


						return normalize_answer(lowerCamelCase						) == normalize_answer(lowerCamelCase						)

def 						lowerCAmelCase							(	lowerCamelCase ,     lowerCamelCase ,     lowerCamelCase						):
						'''simple docstring'''


						snake_case__    :   int							     =   []
						for ground_truth in ground_truths:
												snake_case__    :   Optional[Any]							     =   metric_fn(lowerCamelCase ,     lowerCamelCase						)
												scores_for_ground_truths.append(lowerCamelCase						)
						return max(lowerCamelCase						)

def 						lowerCAmelCase							(	lowerCamelCase ,     lowerCamelCase						):
						'''simple docstring'''


						snake_case__    :   Union[str, Any]							     =   0
						snake_case__    :   Any							     =   []
						for passage in dataset:
												for qa in passage["qas"]:
																		total += 1
																		if qa["id"] not in predictions:
																								snake_case__    :   int							     =   f'''Unanswered question {qa['id']} will receive score 0.'''
																								print(lowerCamelCase ,     file=sys.stderr						)
																								continue

																		snake_case__    :   Any							     =   [x['''text'''] for x in qa['''answers''']]
																		snake_case__    :   Dict							     =   predictions[qa['''id''']]

																		snake_case__    :   Union[str, Any]							     =   metric_max_over_ground_truths(lowerCamelCase ,     lowerCamelCase ,     lowerCamelCase						)
																		if int(_exact_match						) == 1:
																								correct_ids.append(qa['''id''']						)
																		exact_match += _exact_match

																		fa += metric_max_over_ground_truths(lowerCamelCase ,     lowerCamelCase ,     lowerCamelCase						)

						snake_case__    :   str							     =   exact_match / total
						snake_case__    :   Optional[int]							     =   fa / total

						return {"exact_match": exact_match, "f1": fa}, correct_ids


if __name__ == "__main__":
							_lowercase		 =      """1.0"""
							_lowercase		 =      argparse.ArgumentParser("""Official evaluation script for ReCoRD v1.0.""")
							parser.add_argument("""data_file""", help="""The dataset file in JSON format.""")
							parser.add_argument("""pred_file""", help="""The model prediction file in JSON format.""")
							parser.add_argument("""--output_correct_ids""", action="""store_true""", help="""Output the correctly answered query IDs.""")
							_lowercase		 =      parser.parse_args()

							with open(args.data_file) as data_file:
														_lowercase		 =      json.load(data_file)
														if dataset_json["version"] != expected_version:
																					print(
																					    f"""Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json["version"]}""",
																					    file=sys.stderr,
																					)
														_lowercase		 =      dataset_json["""data"""]

							with open(args.pred_file) as pred_file:
														_lowercase		 =      json.load(pred_file)

							_lowercase ,_lowercase		 =      evaluate(dataset, predictions)

							if args.output_correct_ids:
														print(f"""Output {len(correct_ids)} correctly answered question IDs.""")
														with open("""correct_ids.json""", """w""") as f:
																					json.dump(correct_ids, f)


