





'''simple docstring'''

import sacrebleu as scb
from packaging import version

import datasets


lowercase_         =			"""\
@inproceedings{post-2018-call,
    title = \"A Call for Clarity in Reporting {BLEU} Scores\",
    author = \"Post, Matt\",
    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",
    month = oct,
    year = \"2018\",
    address = \"Belgium, Brussels\",
    publisher = \"Association for Computational Linguistics\",
    url = \"https://www.aclweb.org/anthology/W18-6319\",
    pages = \"186--191\",
}
"""

lowercase_         =			"""\
SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.
Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.
It also knows all the standard test sets and handles downloading, processing, and tokenization for you.

See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information.
"""

lowercase_         =			"""
Produces BLEU scores along with its sufficient statistics
from a source against one or more references.

Args:
    predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens.
    references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length).
    smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are:
        - `'none'`: no smoothing
        - `'floor'`: increment zero counts
        - `'add-k'`: increment num/denom by k for n>1
        - `'exp'`: exponential decay
    smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`).
    tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are:
        - `'none'`: No tokenization.
        - `'zh'`: Chinese tokenization.
        - `'13a'`: mimics the `mteval-v13a` script from Moses.
        - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses
        - `'char'`: Language-agnostic character-level tokenization.
        - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3).
    lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`.
    force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`.
    use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`.

Returns:
    'score': BLEU score,
    'counts': Counts,
    'totals': Totals,
    'precisions': Precisions,
    'bp': Brevity penalty,
    'sys_len': predictions length,
    'ref_len': reference length,

Examples:

    Example 1:
        >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]
        >>> references = [[\"hello there general kenobi\", \"hello there !\"], [\"foo bar foobar\", \"foo bar foobar\"]]
        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")
        >>> results = sacrebleu.compute(predictions=predictions, references=references)
        >>> print(list(results.keys()))
        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
        >>> print(round(results[\"score\"], 1))
        100.0

    Example 2:
        >>> predictions = [\"hello there general kenobi\",
        ...                 \"on our way to ankh morpork\"]
        >>> references = [[\"hello there general kenobi\", \"hello there !\"],
        ...                 [\"goodbye ankh morpork\", \"ankh morpork\"]]
        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")
        >>> results = sacrebleu.compute(predictions=predictions,
        ...                             references=references)
        >>> print(list(results.keys()))
        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
        >>> print(round(results[\"score\"], 1))
        39.8
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,   _KWARGS_DESCRIPTION	)
class  a_					(    datasets.Metric	):

				'''simple docstring'''


				def       snake_case_(		self	)		->       Union[str, Any]:
								if version.parse(scb.__version__	) < version.parse("""1.4.12"""	):
												raise ImportWarning(
												    """To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"""
												    """You can install it with `pip install \"sacrebleu>=1.4.12\"`."""	)
								return datasets.MetricInfo(
								    description=_DESCRIPTION			,		citation=_CITATION			,		homepage="""https://github.com/mjpost/sacreBLEU"""			,		inputs_description=_KWARGS_DESCRIPTION			,		features=datasets.Features(
								        {
								            """predictions""": datasets.Value("""string"""			,		id="""sequence"""	),
								            """references""": datasets.Sequence(datasets.Value("""string"""			,		id="""sequence"""	)			,		id="""references"""	),
								        }	)			,		codebase_urls=["""https://github.com/mjpost/sacreBLEU"""]			,		reference_urls=[
								        """https://github.com/mjpost/sacreBLEU""",
								        """https://en.wikipedia.org/wiki/BLEU""",
								        """https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213""",
								    ]			,		)





				def       snake_case_(		self			,		A			,		A			,		A="exp"			,		A=None			,		A=False			,		A=False			,		A=None			,		A=False			,		)		->       Optional[Any]:
								_SCREAMING_SNAKE_CASE     =      len(references[0]	)
								if any(len(A	) != references_per_prediction for refs in references	):
												raise ValueError("""Sacrebleu requires the same number of references for each prediction"""	)
								_SCREAMING_SNAKE_CASE     =      [[refs[i] for refs in references] for i in range(A	)]
								_SCREAMING_SNAKE_CASE     =      scb.corpus_bleu(
								    A			,		A			,		smooth_method=A			,		smooth_value=A			,		force=A			,		lowercase=A			,		use_effective_order=A			,		**({"""tokenize""": tokenize} if tokenize else {})			,		)
								_SCREAMING_SNAKE_CASE     =      {
								    """score""": output.score,
								    """counts""": output.counts,
								    """totals""": output.totals,
								    """precisions""": output.precisions,
								    """bp""": output.bp,
								    """sys_len""": output.sys_len,
								    """ref_len""": output.ref_len,
								}
								return output_dict










'''simple docstring'''

import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

lowercase_         =			logging.get_logger(__name__)

lowercase_         =			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

lowercase_         =			{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

lowercase_         =			{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

lowercase_         =			{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}





def lowerCamelCase						(					)						->Dict:
    _SCREAMING_SNAKE_CASE     =      (
        list(range(ord("""!"""   ) ,	ord("""~"""   ) + 1   )   ) + list(range(ord("""¡"""   ) ,	ord("""¬"""   ) + 1   )   ) + list(range(ord("""®"""   ) ,	ord("""ÿ"""   ) + 1   )   )
    )
    _SCREAMING_SNAKE_CASE     =      bs[:]
    _SCREAMING_SNAKE_CASE     =      0
    for b in range(2**8   ):
        if b not in bs:
            bs.append(__lowerCamelCase   )
            cs.append(2**8 + n   )
            n += 1
    _SCREAMING_SNAKE_CASE     =      [chr(__lowerCamelCase   ) for n in cs]
    return dict(zip(__lowerCamelCase ,	__lowerCamelCase   )   )





def lowerCamelCase						(					__lowerCamelCase			:   Dict   )						->Optional[int]:
    _SCREAMING_SNAKE_CASE     =      set()
    _SCREAMING_SNAKE_CASE     =      word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)   )
        _SCREAMING_SNAKE_CASE     =      char
    return pairs






class  a_					(    snake_case_	):

    '''simple docstring'''
    UpperCamelCase			   =			VOCAB_FILES_NAMES
    UpperCamelCase			   =			PRETRAINED_VOCAB_FILES_MAP
    UpperCamelCase			   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCamelCase			   =			['''input_ids''', '''attention_mask''', '''token_type_ids''']


    def __init__(		self			,		A			,		A			,		A="replace"			,		A="[CLS]"			,		A="[SEP]"			,		A="[SEP]"			,		A="[CLS]"			,		A="[UNK]"			,		A="[PAD]"			,		A="[MASK]"			,		A=False			,		A=False			,		**A			,		)		->       Dict:
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else bos_token
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else eos_token
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else sep_token
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else cls_token
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else unk_token
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        _SCREAMING_SNAKE_CASE     =      AddedToken(A			,		lstrip=A			,		rstrip=A	) if isinstance(A			,		A	) else mask_token

        super().__init__(
            errors=A			,		bos_token=A			,		eos_token=A			,		unk_token=A			,		sep_token=A			,		cls_token=A			,		pad_token=A			,		mask_token=A			,		add_prefix_space=A			,		add_bos_token=A			,		**A			,		)
        _SCREAMING_SNAKE_CASE     =      add_bos_token

        with open(A			,		encoding="""utf-8"""	) as vocab_handle:
            _SCREAMING_SNAKE_CASE     =      json.load(A	)
        _SCREAMING_SNAKE_CASE     =      {v: k for k, v in self.encoder.items()}
        _SCREAMING_SNAKE_CASE     =      errors  # how to handle errors in decoding
        _SCREAMING_SNAKE_CASE     =      bytes_to_unicode()
        _SCREAMING_SNAKE_CASE     =      {v: k for k, v in self.byte_encoder.items()}
        with open(A			,		encoding="""utf-8"""	) as merges_handle:
            _SCREAMING_SNAKE_CASE     =      merges_handle.read().split("""\n"""	)[1:-1]
        _SCREAMING_SNAKE_CASE     =      [tuple(merge.split()	) for merge in bpe_merges]
        _SCREAMING_SNAKE_CASE     =      dict(zip(A			,		range(len(A	)	)	)	)
        _SCREAMING_SNAKE_CASE     =      {}
        _SCREAMING_SNAKE_CASE     =      add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        _SCREAMING_SNAKE_CASE     =      re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""	)


    @property
    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
    def       snake_case_(		self	)		->       List[str]:
        return len(self.encoder	)


    def       snake_case_(		self	)		->       Optional[int]:
        return dict(self.encoder			,		**self.added_tokens_encoder	)


    def       snake_case_(		self			,		A	)		->       Optional[Any]:
        if token in self.cache:
            return self.cache[token]
        _SCREAMING_SNAKE_CASE     =      tuple(A	)
        _SCREAMING_SNAKE_CASE     =      get_pairs(A	)

        if not pairs:
            return token

        while True:
            _SCREAMING_SNAKE_CASE     =      min(A			,		key=lambda A	: self.bpe_ranks.get(A			,		float("""inf"""	)	)	)
            if bigram not in self.bpe_ranks:
                break
            _SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     =      bigram
            _SCREAMING_SNAKE_CASE     =      []
            _SCREAMING_SNAKE_CASE     =      0
            while i < len(A	):
                try:
                    _SCREAMING_SNAKE_CASE     =      word.index(A			,		A	)
                except ValueError:
                    new_word.extend(word[i:]	)
                    break
                else:
                    new_word.extend(word[i:j]	)
                    _SCREAMING_SNAKE_CASE     =      j

                if word[i] == first and i < len(A	) - 1 and word[i + 1] == second:
                    new_word.append(first + second	)
                    i += 2
                else:
                    new_word.append(word[i]	)
                    i += 1
            _SCREAMING_SNAKE_CASE     =      tuple(A	)
            _SCREAMING_SNAKE_CASE     =      new_word
            if len(A	) == 1:
                break
            else:
                _SCREAMING_SNAKE_CASE     =      get_pairs(A	)
        _SCREAMING_SNAKE_CASE     =      """ """.join(A	)
        _SCREAMING_SNAKE_CASE     =      word
        return word


    def       snake_case_(		self			,		A			,		A = None	)		->       List[int]:
        if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        _SCREAMING_SNAKE_CASE     =      [self.cls_token_id]
        _SCREAMING_SNAKE_CASE     =      [self.sep_token_id]
        return cls + token_ids_a + sep + token_ids_a + sep


    def       snake_case_(		self			,		A			,		A = None			,		A = False	)		->       List[int]:
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=A			,		token_ids_a=A			,		already_has_special_tokens=A	)

        if token_ids_a is None:
            return [1] + ([0] * len(A	)) + [1]
        return [1] + ([0] * len(A	)) + [1] + ([0] * len(A	)) + [1]


    def       snake_case_(		self			,		A			,		A = None	)		->       List[int]:
        _SCREAMING_SNAKE_CASE     =      [self.sep_token_id]
        _SCREAMING_SNAKE_CASE     =      [self.cls_token_id]

        if token_ids_a is None:
            return len(cls + token_ids_a + sep	) * [0]
        return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]


    def       snake_case_(		self			,		A	)		->       Union[str, Any]:
        _SCREAMING_SNAKE_CASE     =      []
        for token in re.findall(self.pat			,		A	):
            _SCREAMING_SNAKE_CASE     =      """""".join(
                self.byte_encoder[b] for b in token.encode("""utf-8"""	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A	).split(""" """	)	)
        return bpe_tokens


    def       snake_case_(		self			,		A	)		->       List[Any]:
        return self.encoder.get(A			,		self.encoder.get(self.unk_token	)	)


    def       snake_case_(		self			,		A	)		->       Dict:
        return self.decoder.get(A	)


    def       snake_case_(		self			,		A	)		->       List[Any]:
        _SCREAMING_SNAKE_CASE     =      """""".join(A	)
        _SCREAMING_SNAKE_CASE     =      bytearray([self.byte_decoder[c] for c in text]	).decode("""utf-8"""			,		errors=self.errors	)
        return text


    def       snake_case_(		self			,		A			,		A = None	)		->       Tuple[str]:
        if not os.path.isdir(A	):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
            return
        _SCREAMING_SNAKE_CASE     =      os.path.join(
            A			,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
        _SCREAMING_SNAKE_CASE     =      os.path.join(
            A			,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]	)

        with open(A			,		"""w"""			,		encoding="""utf-8"""	) as f:
            f.write(json.dumps(self.encoder			,		indent=2			,		sort_keys=A			,		ensure_ascii=A	) + """\n"""	)

        _SCREAMING_SNAKE_CASE     =      0
        with open(A			,		"""w"""			,		encoding="""utf-8"""	) as writer:
            writer.write("""#version: 0.2\n"""	)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,		key=lambda A	: kv[1]	):
                if index != token_index:
                    logger.warning(
                        f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                        """ Please check that the tokenizer is not corrupted!"""	)
                    _SCREAMING_SNAKE_CASE     =      token_index
                writer.write(""" """.join(A	) + """\n"""	)
                index += 1

        return vocab_file, merge_file


    def       snake_case_(		self			,		A			,		A=False			,		**A	)		->       str:
        _SCREAMING_SNAKE_CASE     =      kwargs.pop("""add_prefix_space"""			,		self.add_prefix_space	)
        if (is_split_into_words or add_prefix_space) and (len(A	) > 0 and not text[0].isspace()):
            _SCREAMING_SNAKE_CASE     =      """ """ + text
        return (text, kwargs)





    def       snake_case_(		self			,		A	)		->       List[int]:
        _SCREAMING_SNAKE_CASE     =      []
        for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(A			,		add_special_tokens=A	) + [self.eos_token_id]	)
        if len(A	) > self.model_max_length:
            _SCREAMING_SNAKE_CASE     =      input_ids[-self.model_max_length :]
        return input_ids



