





import sacrebleu as scb
from packaging import version

import datasets


lowercase__    :					Optional[Any]					=						"\\n@inproceedings{post-2018-call,\n    title = \"A Call for Clarity in Reporting {BLEU} Scores\",\n    author = \"Post, Matt\",\n    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",\n    month = oct,\n    year = \"2018\",\n    address = \"Belgium, Brussels\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://www.aclweb.org/anthology/W18-6319\",\n    pages = \"186--191\",\n}\n"

lowercase__    :					Any					=						"\\nSacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.\nInspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.\nIt also knows all the standard test sets and handles downloading, processing, and tokenization for you.\n\nSee the [README.md] file at https://github.com/mjpost/sacreBLEU for more information.\n"

lowercase__    :					Any					=						"\nProduces BLEU scores along with its sufficient statistics\nfrom a source against one or more references.\n\nArgs:\n    predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens.\n    references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length).\n    smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are:\n        - `'none'`: no smoothing\n        - `'floor'`: increment zero counts\n        - `'add-k'`: increment num/denom by k for n>1\n        - `'exp'`: exponential decay\n    smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`).\n    tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are:\n        - `'none'`: No tokenization.\n        - `'zh'`: Chinese tokenization.\n        - `'13a'`: mimics the `mteval-v13a` script from Moses.\n        - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses\n        - `'char'`: Language-agnostic character-level tokenization.\n        - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3).\n    lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`.\n    force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`.\n    use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`.\n\nReturns:\n    'score': BLEU score,\n    'counts': Counts,\n    'totals': Totals,\n    'precisions': Precisions,\n    'bp': Brevity penalty,\n    'sys_len': predictions length,\n    'ref_len': reference length,\n\nExamples:\n\n    Example 1:\n        >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]\n        >>> references = [[\"hello there general kenobi\", \"hello there !\"], [\"foo bar foobar\", \"foo bar foobar\"]]\n        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")\n        >>> results = sacrebleu.compute(predictions=predictions, references=references)\n        >>> print(list(results.keys()))\n        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']\n        >>> print(round(results[\"score\"], 1))\n        100.0\n\n    Example 2:\n        >>> predictions = [\"hello there general kenobi\",\n        ...                 \"on our way to ankh morpork\"]\n        >>> references = [[\"hello there general kenobi\", \"hello there !\"],\n        ...                 [\"goodbye ankh morpork\", \"ankh morpork\"]]\n        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")\n        >>> results = sacrebleu.compute(predictions=predictions,\n        ...                             references=references)\n        >>> print(list(results.keys()))\n        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']\n        >>> print(round(results[\"score\"], 1))\n        39.8\n"
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,       _KWARGS_DESCRIPTION      )
class 		UpperCAmelCase       (    datasets.Metric      ):







						'''simple docstring'''





						def 	snake_case__						(						self							:					Dict						):






							"""simple docstring"""






							if version.parse(scb.__version__						) < version.parse("1.4.12"						):
								raise ImportWarning(
								    "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n"
								    "You can install it with `pip install \"sacrebleu>=1.4.12\"`."						)
							return datasets.MetricInfo(
							    description=_DESCRIPTION	,     citation=_CITATION	,     homepage="https://github.com/mjpost/sacreBLEU"	,     inputs_description=_KWARGS_DESCRIPTION	,     features=datasets.Features(
							        {
							            "predictions": datasets.Value("string"	,     id="sequence"						),
							            "references": datasets.Sequence(datasets.Value("string"	,     id="sequence"						)	,     id="references"						),
							        }						)	,     codebase_urls=["https://github.com/mjpost/sacreBLEU"]	,     reference_urls=[
							        "https://github.com/mjpost/sacreBLEU",
							        "https://en.wikipedia.org/wiki/BLEU",
							        "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213",
							    ]	,     )






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Optional[int]	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					str="exp"	,     __lowercase							:					List[Any]=None	,     __lowercase							:					str=False	,     __lowercase							:					Union[str, Any]=False	,     __lowercase							:					List[str]=None	,     __lowercase							:					Union[str, Any]=False	,     ):






							"""simple docstring"""






							snake_case_		   =					len(references[0]						)
							if any(len(__snake_case						) != references_per_prediction for refs in references						):
								raise ValueError("Sacrebleu requires the same number of references for each prediction"						)
							snake_case_		   =					[[refs[i] for refs in references] for i in range(__snake_case						)]
							snake_case_		   =					scb.corpus_bleu(
							    __snake_case	,     __snake_case	,     smooth_method=__snake_case	,     smooth_value=__snake_case	,     force=__snake_case	,     lowercase=__snake_case	,     use_effective_order=__snake_case	,     **({"tokenize": tokenize} if tokenize else {})	,     )
							snake_case_		   =					{
							    '''score''': output.score,
							    '''counts''': output.counts,
							    '''totals''': output.totals,
							    '''precisions''': output.precisions,
							    '''bp''': output.bp,
							    '''sys_len''': output.sys_len,
							    '''ref_len''': output.ref_len,
							}
							return output_dict






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase__    :					List[str]					=						logging.get_logger(__name__)

lowercase__    :					Tuple					=						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowercase__    :					Any					=						{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

lowercase__    :					Tuple					=						{
    "openai-gpt": 512,
}


def 						lowerCamelCase__   (		_A							):







	'''simple docstring'''





	snake_case_		   =					text.strip()
	if not text:
		return []
	snake_case_		   =					text.split()
	return tokens
class 		UpperCAmelCase       (    UpperCAmelCase__      ):







						'''simple docstring'''





						def __init__(						self							:					Dict	,     __lowercase							:					str=True	,     __lowercase							:					Union[str, Any]=None	,     __lowercase							:					str=True	,     __lowercase							:					Dict=None	,     __lowercase							:					str=True	,     ):






							"""simple docstring"""






							if never_split is None:
								snake_case_		   =					[]
							snake_case_		   =					do_lower_case
							snake_case_		   =					set(__lowercase						)
							snake_case_		   =					tokenize_chinese_chars
							snake_case_		   =					strip_accents
							snake_case_		   =					do_split_on_punc






						def 	snake_case__						(						self							:					Any	,     __lowercase							:					Any	,     __lowercase							:					Optional[int]=None						):






							"""simple docstring"""






							snake_case_		   =					self.never_split.union(set(__lowercase						)						) if never_split else self.never_split
							snake_case_		   =					self._clean_text(__lowercase						)

							# This was added on November 1st, 2018 for the multilingual and Chinese
							# models. This is also applied to the English models now, but it doesn't
							# matter since the English models were not trained on any Chinese data
							# and generally don't have any Chinese data in them (there are Chinese
							# characters in the vocabulary because Wikipedia does have some Chinese
							# words in the English Wikipedia.).
							if self.tokenize_chinese_chars:
								snake_case_		   =					self._tokenize_chinese_chars(__lowercase						)
							# prevents treating the same character with different unicode codepoints as different characters
							snake_case_		   =					unicodedata.normalize("NFC"	,     __lowercase						)
							snake_case_		   =					whitespace_tokenize(__lowercase						)
							snake_case_		   =					[]
							for token in orig_tokens:
								if token not in never_split:
									if self.do_lower_case:
										snake_case_		   =					token.lower()
										if self.strip_accents is not False:
											snake_case_		   =					self._run_strip_accents(__lowercase						)
									elif self.strip_accents:
										snake_case_		   =					self._run_strip_accents(__lowercase						)
								split_tokens.extend(self._run_split_on_punc(__lowercase	,     __lowercase						)						)

							snake_case_		   =					whitespace_tokenize(" ".join(__lowercase						)						)
							return output_tokens






						def 	snake_case__						(						self							:					str	,     __lowercase							:					List[str]						):






							"""simple docstring"""






							snake_case_		   =					unicodedata.normalize("NFD"	,     __lowercase						)
							snake_case_		   =					[]
							for char in text:
								snake_case_		   =					unicodedata.category(__lowercase						)
								if cat == "Mn":
									continue
								output.append(__lowercase						)
							return "".join(__lowercase						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Optional[int]=None						):






							"""simple docstring"""






							if not self.do_split_on_punc or (never_split is not None and text in never_split):
								return [text]
							snake_case_		   =					list(__lowercase						)
							snake_case_		   =					0
							snake_case_		   =					True
							snake_case_		   =					[]
							while i < len(__lowercase						):
								snake_case_		   =					chars[i]
								if _is_punctuation(__lowercase						):
									output.append([char]						)
									snake_case_		   =					True
								else:
									if start_new_word:
										output.append([]						)
									snake_case_		   =					False
									output[-1].append(__lowercase						)
								i += 1

							return ["".join(__lowercase						) for x in output]






						def 	snake_case__						(						self							:					Optional[int]	,     __lowercase							:					str						):






							"""simple docstring"""






							snake_case_		   =					[]
							for char in text:
								snake_case_		   =					ord(__lowercase						)
								if self._is_chinese_char(__lowercase						):
									output.append(" "						)
									output.append(__lowercase						)
									output.append(" "						)
								else:
									output.append(__lowercase						)
							return "".join(__lowercase						)






						def 	snake_case__						(						self							:					Union[str, Any]	,     __lowercase							:					str						):






							"""simple docstring"""






							if (
							    (cp >= 0X4_e00 and cp <= 0X9_fff)
							    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
							    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
							    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
							    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
							    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
							    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
							    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
							):  #
								return True

							return False






						def 	snake_case__						(						self							:					int	,     __lowercase							:					Dict						):






							"""simple docstring"""






							snake_case_		   =					[]
							for char in text:
								snake_case_		   =					ord(__lowercase						)
								if cp == 0 or cp == 0Xf_ffd or _is_control(__lowercase						):
									continue
								if _is_whitespace(__lowercase						):
									output.append(" "						)
								else:
									output.append(__lowercase						)
							return "".join(__lowercase						)







def 						lowerCamelCase__   (		_A							):







	'''simple docstring'''





	snake_case_		   =					set()
	snake_case_		   =					word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)							)
		snake_case_		   =					char
	return pairs


def 						lowerCamelCase__   (		_A							):







	'''simple docstring'''





	snake_case_		   =					text.replace("—"					,     "-"							)
	snake_case_		   =					text.replace("–"					,     "-"							)
	snake_case_		   =					text.replace("―"					,     "-"							)
	snake_case_		   =					text.replace("…"					,     "..."							)
	snake_case_		   =					text.replace("´"					,     "'"							)
	snake_case_		   =					re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"					,     R" \1 "					,     _A							)
	snake_case_		   =					re.sub(R"\s*\n\s*"					,     " \n "					,     _A							)
	snake_case_		   =					re.sub(R"[^\S\n]+"					,     " "					,     _A							)
	return text.strip()






class 		UpperCAmelCase       (    UpperCAmelCase__      ):







						'''simple docstring'''





						lowerCAmelCase_							=       VOCAB_FILES_NAMES
						lowerCAmelCase_							=       PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase_							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase_							=       ['''input_ids''', '''attention_mask''']






						def __init__(						self							:					Dict	,     __lowercase							:					Union[str, Any]	,     __lowercase							:					Tuple	,     __lowercase							:					Any="<unk>"	,     **__lowercase							:					List[Any]						):






							"""simple docstring"""






							super().__init__(unk_token=__lowercase	,     **__lowercase						)

							try:
								import ftfy
								from spacy.lang.en import English

								snake_case_		   =					English()
								snake_case_		   =					_nlp.tokenizer
								snake_case_		   =					ftfy.fix_text
							except ImportError:
								logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."						)
								snake_case_		   =					BasicTokenizer(do_lower_case=__lowercase						)
								snake_case_		   =					None

							with open(__lowercase	,     encoding="utf-8"						) as vocab_handle:
								snake_case_		   =					json.load(__lowercase						)
							snake_case_		   =					{v: k for k, v in self.encoder.items()}
							with open(__lowercase	,     encoding="utf-8"						) as merges_handle:
								snake_case_		   =					merges_handle.read().split("\n"						)[1:-1]
							snake_case_		   =					[tuple(merge.split()						) for merge in merges]
							snake_case_		   =					dict(zip(__lowercase	,     range(len(__lowercase						)						)						)						)
							snake_case_		   =					{}






						@property
						def 	snake_case__						(						self							:					List[Any]						):






							"""simple docstring"""






							return True






						@property
						def 	snake_case__						(						self							:					Union[str, Any]						):






							"""simple docstring"""






							return len(self.encoder						)






						def 	snake_case__						(						self							:					str						):






							"""simple docstring"""






							return dict(self.encoder	,     **self.added_tokens_encoder						)






						def 	snake_case__						(						self							:					Optional[Any]	,     __lowercase							:					int						):






							"""simple docstring"""






							snake_case_		   =					tuple(token[:-1]						) + (token[-1] + "</w>",)
							if token in self.cache:
								return self.cache[token]
							snake_case_		   =					get_pairs(__lowercase						)

							if not pairs:
								return token + "</w>"

							while True:
								snake_case_		   =					min(__lowercase	,     key=lambda __lowercase						: self.bpe_ranks.get(__lowercase	,     float("inf"						)						)						)
								if bigram not in self.bpe_ranks:
									break
								snake_case_   ,				snake_case_		   =					bigram
								snake_case_		   =					[]
								snake_case_		   =					0
								while i < len(__lowercase						):
									try:
										snake_case_		   =					word.index(__lowercase	,     __lowercase						)
									except ValueError:
										new_word.extend(word[i:]						)
										break
									else:
										new_word.extend(word[i:j]						)
										snake_case_		   =					j

									if word[i] == first and i < len(__lowercase						) - 1 and word[i + 1] == second:
										new_word.append(first + second						)
										i += 2
									else:
										new_word.append(word[i]						)
										i += 1
								snake_case_		   =					tuple(__lowercase						)
								snake_case_		   =					new_word
								if len(__lowercase						) == 1:
									break
								else:
									snake_case_		   =					get_pairs(__lowercase						)
							snake_case_		   =					" ".join(__lowercase						)
							if word == "\n  </w>":
								snake_case_		   =					"\n</w>"
							snake_case_		   =					word
							return word






						def 	snake_case__						(						self							:					List[str]	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							snake_case_		   =					[]
							if self.fix_text is None:
								# Using BERT's BasicTokenizer
								snake_case_		   =					self.nlp.tokenize(__lowercase						)
								for token in text:
									split_tokens.extend(list(self.bpe(__lowercase						).split(" "						)						)						)
							else:
								# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
								snake_case_		   =					self.nlp(text_standardize(self.fix_text(__lowercase						)						)						)
								for token in text:
									split_tokens.extend(list(self.bpe(token.text.lower()						).split(" "						)						)						)
							return split_tokens






						def 	snake_case__						(						self							:					int	,     __lowercase							:					Tuple						):






							"""simple docstring"""






							return self.encoder.get(__lowercase	,     self.encoder.get(self.unk_token						)						)






						def 	snake_case__						(						self							:					Tuple	,     __lowercase							:					Any						):






							"""simple docstring"""






							return self.decoder.get(__lowercase	,     self.unk_token						)






						def 	snake_case__						(						self							:					str	,     __lowercase							:					List[Any]						):






							"""simple docstring"""






							snake_case_		   =					"".join(__lowercase						).replace("</w>"	,     " "						).strip()
							return out_string






						def 	snake_case__						(						self							:					int	,     __lowercase							:					str	,     __lowercase							:					Optional[str] = None						):






							"""simple docstring"""






							if not os.path.isdir(__lowercase						):
								logger.error(f"Vocabulary path ({save_directory}) should be a directory"						)
								return
							snake_case_		   =					os.path.join(
							    __lowercase	,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
							snake_case_		   =					os.path.join(
							    __lowercase	,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

							with open(__lowercase	,     "w"	,     encoding="utf-8"						) as f:
								f.write(json.dumps(self.encoder	,     indent=2	,     sort_keys=__lowercase	,     ensure_ascii=__lowercase						) + "\n"						)

							snake_case_		   =					0
							with open(__lowercase	,     "w"	,     encoding="utf-8"						) as writer:
								writer.write("#version: 0.2\n"						)
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,     key=lambda __lowercase						: kv[1]						):
									if index != token_index:
										logger.warning(
										    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
										    " Please check that the tokenizer is not corrupted!"						)
										snake_case_		   =					token_index
									writer.write(" ".join(__lowercase						) + "\n"						)
									index += 1

							return vocab_file, merge_file
