
"""simple docstring"""




from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse('3.8'):
 import importlib_metadata
else:
 import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
snake_case			       =		'\\n'

snake_case			       =		'\\n    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.\n    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.\n'

snake_case			       =		'\nCompute XTREME-S evaluation metric associated to each XTREME-S dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    bleu_kwargs: optional dict of keywords to be passed when computing \'bleu\'.\n        Keywords include Dict can be one of \'smooth_method\', \'smooth_value\', \'force\', \'lowercase\',\n        \'tokenize\', \'use_effective_order\'.\n    wer_kwargs: optional dict of keywords to be passed when computing \'wer\' and \'cer\'.\n        Keywords include \'concatenate_texts\'.\nReturns: depending on the XTREME-S task, one or several of:\n    "accuracy": Accuracy - for \'fleurs-lang_id\', \'minds14\'\n    "f1": F1 score - for \'minds14\'\n    "wer": Word error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "cer": Character error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "bleu": BLEU score according to the `sacrebleu` metric - for \'covost2\'\nExamples:\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'mls\')  # \'mls\', \'voxpopuli\', \'fleurs-asr\' or \'babel\'\n    >>> references = ["it is sunny here", "paper and pen are essentials"]\n    >>> predictions = ["it\'s sunny", "paper pen are essential"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'wer\': 0.56, \'cer\': 0.27}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'covost2\')\n    >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"]\n    >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'bleu\': 31.65}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'fleurs-lang_id\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'accuracy\': 0.6}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'minds14\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'f1\': 0.58, \'accuracy\': 0.6}\n'

snake_case			       =		['fleurs-asr', 'mls', 'voxpopuli', 'babel', 'covost2', 'fleurs-lang_id', 'minds14']
snake_case			       =		''

try:
 from jiwer import transforms as tr

 snake_case			       =		True
except ImportError:
 snake_case			       =		False

if _jiwer_available and version.parse(importlib_metadata.version('jiwer')) < version.parse('2.3.0'):




 class 					UpperCamelCase     (	tr.AbstractTransform				):

     """simple docstring"""



     def __init__(    self		,     lowercase__ = " "      )   ->   Dict:
          """simple docstring"""







          SCREAMING_SNAKE_CASE           =     sentence_delimiter


     def 							A (    self		,     lowercase__      )   ->   Any:
          """simple docstring"""







          return list(lowercase__      )




     def 							A (    self		,     lowercase__      )   ->   List[Any]:
          """simple docstring"""







          SCREAMING_SNAKE_CASE           =     []
          for sent_idx, sentence in enumerate(lowercase__      ):
               chars.extend(self.process_string(lowercase__      )      )
               if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(lowercase__      ) - 1:
                    chars.append(self.sentence_delimiter      )
          return chars






 snake_case			       =		tr.Compose(
     [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
 )
elif _jiwer_available:
 snake_case			       =		tr.Compose(
     [
         tr.RemoveMultipleSpaces(),
         tr.Strip(),
         tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
         tr.ReduceToListOfListOfChars(),
     ]
 )
else:
 snake_case			       =		None

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
     return float((preds == labels).mean()      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
     return {
         "f1": float(fa_score(y_true=SCREAMING_SNAKE_CASE_,					y_pred=SCREAMING_SNAKE_CASE_,					average='macro'      )      ),
         "accuracy": simple_accuracy(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ),
     }

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_="exp",					SCREAMING_SNAKE_CASE_=None,					SCREAMING_SNAKE_CASE_=False,					SCREAMING_SNAKE_CASE_=False,					SCREAMING_SNAKE_CASE_=None,					SCREAMING_SNAKE_CASE_=False,					):
     # xtreme-s can only have one label
     SCREAMING_SNAKE_CASE           =     [[label] for label in labels]
     SCREAMING_SNAKE_CASE           =     list(SCREAMING_SNAKE_CASE_      )
     try:
          import sacrebleu as scb
     except ImportError:
          raise ValueError(
              'sacrebleu has to be installed in order to apply the bleu metric for covost2.'
              'You can install it via `pip install sacrebleu`.'      )

     if version.parse(scb.__version__      ) < version.parse('1.4.12'      ):
          raise ImportWarning(
              'To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n'
              'You can install it with `pip install "sacrebleu>=1.4.12"`.'      )

     SCREAMING_SNAKE_CASE           =     len(labels[0]      )
     if any(len(SCREAMING_SNAKE_CASE_      ) != references_per_prediction for refs in labels      ):
          raise ValueError('Sacrebleu requires the same number of references for each prediction'      )
     SCREAMING_SNAKE_CASE           =     [[refs[i] for refs in labels] for i in range(SCREAMING_SNAKE_CASE_      )]
     SCREAMING_SNAKE_CASE           =     scb.corpus_bleu(
         SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					smooth_method=SCREAMING_SNAKE_CASE_,					smooth_value=SCREAMING_SNAKE_CASE_,					force=SCREAMING_SNAKE_CASE_,					lowercase=SCREAMING_SNAKE_CASE_,					use_effective_order=SCREAMING_SNAKE_CASE_,					**({'tokenize': tokenize} if tokenize else {}),					)
     return {"bleu": output.score}

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
     try:
          from jiwer import compute_measures
     except ImportError:
          raise ValueError(
              f'''jiwer has to be installed in order to apply the wer metric for {config_name}.'''
              'You can install it via `pip install jiwer`.'      )

     if concatenate_texts:
          SCREAMING_SNAKE_CASE           =     compute_measures(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )['wer']

          SCREAMING_SNAKE_CASE           =     compute_measures(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					truth_transform=SCREAMING_SNAKE_CASE_,					hypothesis_transform=SCREAMING_SNAKE_CASE_      )['wer']
          return {"wer": wer, "cer": cer}
     else:

          def compute_score(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_="wer"      ):
               SCREAMING_SNAKE_CASE           =     0
               SCREAMING_SNAKE_CASE           =     0
               for prediction, reference in zip(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
                    if score_type == "wer":
                         SCREAMING_SNAKE_CASE           =     compute_measures(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )
                    elif score_type == "cer":
                         SCREAMING_SNAKE_CASE           =     compute_measures(
                             SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					truth_transform=SCREAMING_SNAKE_CASE_,					hypothesis_transform=SCREAMING_SNAKE_CASE_      )
                    incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
                    total += measures["substitutions"] + measures["deletions"] + measures["hits"]
               return incorrect / total

          return {"wer": compute_score(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					'wer'      ), "cer": compute_score(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					'cer'      )}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,				_KWARGS_DESCRIPTION				)
class 					UpperCamelCase     (	datasets.Metric				):

    """simple docstring"""



    def 							A (    self      )   ->   Any:
         """simple docstring"""







         if self.config_name not in _CONFIG_NAMES:
              raise KeyError(f'''You should supply a configuration name selected in {_CONFIG_NAMES}'''      )

         SCREAMING_SNAKE_CASE           =     'int64' if self.config_name in ['fleurs-lang_id', 'minds14'] else 'string'

         return datasets.MetricInfo(
             description=_DESCRIPTION		,     citation=_CITATION		,     inputs_description=_KWARGS_DESCRIPTION		,     features=datasets.Features(
                 {'predictions': datasets.Value(lowercase__      ), 'references': datasets.Value(lowercase__      )}      )		,     codebase_urls=[]		,     reference_urls=[]		,     format='numpy'		,     )




    def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__=None		,     lowercase__=None      )   ->   Union[str, Any]:
         """simple docstring"""







         SCREAMING_SNAKE_CASE           =     bleu_kwargs if bleu_kwargs is not None else {}
         SCREAMING_SNAKE_CASE           =     wer_kwargs if wer_kwargs is not None else {}

         if self.config_name == "fleurs-lang_id":
              return {"accuracy": simple_accuracy(lowercase__		,     lowercase__      )}
         elif self.config_name == "minds14":
              return fa_and_simple_accuracy(lowercase__		,     lowercase__      )
         elif self.config_name == "covost2":
              SCREAMING_SNAKE_CASE           =     bleu_kwargs.pop('smooth_method'		,     'exp'      )
              SCREAMING_SNAKE_CASE           =     bleu_kwargs.pop('smooth_value'		,     lowercase__      )
              SCREAMING_SNAKE_CASE           =     bleu_kwargs.pop('force'		,     lowercase__      )
              SCREAMING_SNAKE_CASE           =     bleu_kwargs.pop('lowercase'		,     lowercase__      )
              SCREAMING_SNAKE_CASE           =     bleu_kwargs.pop('tokenize'		,     lowercase__      )
              SCREAMING_SNAKE_CASE           =     bleu_kwargs.pop('use_effective_order'		,     lowercase__      )
              return bleu(
                  preds=lowercase__		,     labels=lowercase__		,     smooth_method=lowercase__		,     smooth_value=lowercase__		,     force=lowercase__		,     lowercase=lowercase__		,     tokenize=lowercase__		,     use_effective_order=lowercase__		,     )
         elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
              SCREAMING_SNAKE_CASE           =     wer_kwargs.pop('concatenate_texts'		,     lowercase__      )
              return wer_and_cer(lowercase__		,     lowercase__		,     lowercase__		,     self.config_name      )
         else:
              raise KeyError(f'''You should supply a configuration name selected in {_CONFIG_NAMES}'''      )


"""simple docstring"""




import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available


if is_torch_available():
	from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
	snake_case			       =		False

if is_vision_available():
	from PIL import Image

	from transformers import (
	    AutoProcessor,
	    PixaStructImageProcessor,
	    PixaStructProcessor,
	    PreTrainedTokenizerFast,
	    TaTokenizer,
	)




@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11  ,				reason="`Pix2StructImageProcessor` requires `torch>=1.11.0`."  ,				)
@require_vision
@require_torch
class 					UpperCamelCase     (	unittest.TestCase				):

				"""simple docstring"""



				def 							A (    self      )   ->   Any:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     tempfile.mkdtemp()

									SCREAMING_SNAKE_CASE           =     PixaStructImageProcessor()
									SCREAMING_SNAKE_CASE           =     TaTokenizer.from_pretrained('t5-small'      )

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(lowercase__		,     lowercase__      )

									processor.save_pretrained(self.tmpdirname      )


				def 							A (    self		,     **lowercase__      )   ->   Dict:
									"""simple docstring"""







									return AutoProcessor.from_pretrained(self.tmpdirname		,     **lowercase__      ).tokenizer


				def 							A (    self		,     **lowercase__      )   ->   Union[str, Any]:
									"""simple docstring"""







									return AutoProcessor.from_pretrained(self.tmpdirname		,     **lowercase__      ).image_processor


				def 							A (    self      )   ->   Optional[int]:
									"""simple docstring"""







									shutil.rmtree(self.tmpdirname      )


				def 							A (    self      )   ->   int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     [np.random.randint(255		,     size=(3, 30, 400)		,     dtype=np.uinta      )]

									SCREAMING_SNAKE_CASE           =     [Image.fromarray(np.moveaxis(lowercase__		,     0		,     -1      )      ) for x in image_inputs]

									return image_inputs


				def 							A (    self      )   ->   int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=self.get_tokenizer()		,     image_processor=self.get_image_processor()      )
									processor.save_pretrained(self.tmpdirname      )

									SCREAMING_SNAKE_CASE           =     self.get_tokenizer(bos_token='(BOS)'		,     eos_token='(EOS)'      )
									SCREAMING_SNAKE_CASE           =     self.get_image_processor(do_normalize=lowercase__		,     padding_value=1.0      )

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor.from_pretrained(
									    self.tmpdirname		,     bos_token='(BOS)'		,     eos_token='(EOS)'		,     do_normalize=lowercase__		,     padding_value=1.0      )

									self.assertEqual(processor.tokenizer.get_vocab()		,     tokenizer_add_kwargs.get_vocab()      )
									self.assertIsInstance(processor.tokenizer		,     lowercase__      )

									self.assertEqual(processor.image_processor.to_json_string()		,     image_processor_add_kwargs.to_json_string()      )
									self.assertIsInstance(processor.image_processor		,     lowercase__      )


				def 							A (    self      )   ->   int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.get_image_processor()
									SCREAMING_SNAKE_CASE           =     self.get_tokenizer()

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=lowercase__		,     image_processor=lowercase__      )

									SCREAMING_SNAKE_CASE           =     self.prepare_image_inputs()

									SCREAMING_SNAKE_CASE           =     image_processor(lowercase__		,     return_tensors='np'      )
									SCREAMING_SNAKE_CASE           =     processor(images=lowercase__		,     return_tensors='np'      )

									for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()		,     input_processor[key].sum()		,     delta=1E-2      )


				def 							A (    self      )   ->   List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.get_image_processor()
									SCREAMING_SNAKE_CASE           =     self.get_tokenizer()

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=lowercase__		,     image_processor=lowercase__      )

									SCREAMING_SNAKE_CASE           =     'lower newer'

									SCREAMING_SNAKE_CASE           =     processor(text=lowercase__      )

									SCREAMING_SNAKE_CASE           =     tokenizer(lowercase__		,     return_token_type_ids=lowercase__		,     add_special_tokens=lowercase__      )

									for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]		,     encoded_processor[key]      )


				def 							A (    self      )   ->   List[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.get_image_processor()
									SCREAMING_SNAKE_CASE           =     self.get_tokenizer()

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=lowercase__		,     image_processor=lowercase__      )

									SCREAMING_SNAKE_CASE           =     'lower newer'
									SCREAMING_SNAKE_CASE           =     self.prepare_image_inputs()

									SCREAMING_SNAKE_CASE           =     processor(text=lowercase__		,     images=lowercase__      )

									self.assertListEqual(
									    list(inputs.keys()      )		,     ['flattened_patches', 'attention_mask', 'decoder_attention_mask', 'decoder_input_ids']      )

									# test if it raises when no input is passed
									with pytest.raises(lowercase__      ):
														processor()


				def 							A (    self      )   ->   Union[str, Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.get_image_processor()
									SCREAMING_SNAKE_CASE           =     self.get_tokenizer()

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=lowercase__		,     image_processor=lowercase__      )

									SCREAMING_SNAKE_CASE           =     'lower newer'
									SCREAMING_SNAKE_CASE           =     self.prepare_image_inputs()

									SCREAMING_SNAKE_CASE           =     processor(text=lowercase__		,     images=lowercase__      )

									SCREAMING_SNAKE_CASE           =     [512, 1024, 2048, 4096]
									SCREAMING_SNAKE_CASE           =     [770, 770, 770, 770]
									# with text
									for i, max_patch in enumerate(lowercase__      ):
														SCREAMING_SNAKE_CASE           =     processor(text=lowercase__		,     images=lowercase__		,     max_patches=lowercase__      )
														self.assertEqual(inputs['flattened_patches'][0].shape[0]		,     lowercase__      )
														self.assertEqual(inputs['flattened_patches'][0].shape[1]		,     expected_hidden_size[i]      )

									# without text input
									for i, max_patch in enumerate(lowercase__      ):
														SCREAMING_SNAKE_CASE           =     processor(images=lowercase__		,     max_patches=lowercase__      )
														self.assertEqual(inputs['flattened_patches'][0].shape[0]		,     lowercase__      )
														self.assertEqual(inputs['flattened_patches'][0].shape[1]		,     expected_hidden_size[i]      )


				def 							A (    self      )   ->   Tuple:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.get_image_processor()
									SCREAMING_SNAKE_CASE           =     self.get_tokenizer()

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=lowercase__		,     image_processor=lowercase__      )

									SCREAMING_SNAKE_CASE           =     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									SCREAMING_SNAKE_CASE           =     processor.batch_decode(lowercase__      )
									SCREAMING_SNAKE_CASE           =     tokenizer.batch_decode(lowercase__      )

									self.assertListEqual(lowercase__		,     lowercase__      )




				def 							A (    self      )   ->   Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.get_image_processor()
									SCREAMING_SNAKE_CASE           =     self.get_tokenizer()

									SCREAMING_SNAKE_CASE           =     PixaStructProcessor(tokenizer=lowercase__		,     image_processor=lowercase__      )

									SCREAMING_SNAKE_CASE           =     'lower newer'
									SCREAMING_SNAKE_CASE           =     self.prepare_image_inputs()

									SCREAMING_SNAKE_CASE           =     processor(text=lowercase__		,     images=lowercase__      )

									# For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"]
									self.assertListEqual(
									    list(inputs.keys()      )		,     ['flattened_patches', 'attention_mask', 'decoder_attention_mask', 'decoder_input_ids']      )

									SCREAMING_SNAKE_CASE           =     processor(text=lowercase__      )

									# For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"]
									self.assertListEqual(list(inputs.keys()      )		,     ['input_ids', 'attention_mask']      )
