



import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
a_       :Tuple				  =				"\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n"

# TODO: Add description of the dataset here
# You can copy an official description
a_       :Union[str, Any]				  =				"\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n"

# TODO: Add a link to an official homepage for the dataset here
a_       :int				  =				""

# TODO: Add the licence for the dataset here if you can find it
a_       :str				  =				""

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
a_       :Optional[int]				  =				{
    "first_domain": "https://huggingface.co/great-new-dataset-first_domain.zip",
    "second_domain": "https://huggingface.co/great-new-dataset-second_domain.zip",
}




class  snake_case__						(	datasets.GeneratorBasedBuilder					):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE            =     datasets.Version("""1.1.0"""					)

     # This is an example of a dataset with multiple configurations.
     # If you don't want/need to define several sub-sets in your dataset,
     # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

     # If you need to make complex sub-parts in the datasets with configurable options
     # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
     # BUILDER_CONFIG_CLASS = MyBuilderConfig

     # You will be able to load one or the other configurations in the following list with
     # data = datasets.load_dataset('my_dataset', 'first_domain')
     # data = datasets.load_dataset('my_dataset', 'second_domain')
     _SCREAMING_SNAKE_CASE            =     [
         datasets.BuilderConfig(name="""first_domain"""    ,					version=lowerCAmelCase_    ,					description="""This part of my dataset covers a first domain"""					),
         datasets.BuilderConfig(name="""second_domain"""    ,					version=lowerCAmelCase_    ,					description="""This part of my dataset covers a second domain"""					),
     ]

     _SCREAMING_SNAKE_CASE            =     """first_domain"""  # It's not mandatory to have a default configuration. Just use one if it make sense.






     def        lowercase_       (	self  :  Optional[int] )       ->List[Any]:
            # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
            if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
                   snake_case__							:     Any            =     datasets.Features(
                       {
                           'sentence': datasets.Value('string' ),
                           'option1': datasets.Value('string' ),
                           'answer': datasets.Value('string' )
                           # These are the features of your dataset like images, labels ...
                       } )
            else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
                   snake_case__							:     Any            =     datasets.Features(
                       {
                           'sentence': datasets.Value('string' ),
                           'option2': datasets.Value('string' ),
                           'second_domain_answer': datasets.Value('string' )
                           # These are the features of your dataset like images, labels ...
                       } )
            return datasets.DatasetInfo(
                # This is the description that will appear on the datasets page.
                description=_DESCRIPTION,			features=_snake_case,			homepage=_HOMEPAGE,			license=_LICENSE,			citation=_CITATION,			)






     def        lowercase_       (	self  :  List[Any],			_snake_case  :  Dict )       ->List[Any]:
            # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
            # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name

            # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
            # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
            # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
            snake_case__							:     Union[str, Any]            =     _URLS[self.config.name]
            snake_case__							:     Optional[int]            =     dl_manager.download_and_extract(_snake_case )
            return [
                datasets.SplitGenerator(
                    name=datasets.Split.TRAIN,			gen_kwargs={
                        'filepath': os.path.join(_snake_case,			'train.jsonl' ),
                        'split': 'train',
                    },			),
                datasets.SplitGenerator(
                    name=datasets.Split.VALIDATION,			gen_kwargs={
                        'filepath': os.path.join(_snake_case,			'dev.jsonl' ),
                        'split': 'dev',
                    },			),
                datasets.SplitGenerator(
                    name=datasets.Split.TEST,			gen_kwargs={
                        'filepath': os.path.join(_snake_case,			'test.jsonl' ),
                        'split': 'test'
                    },			),
            ]






     def        lowercase_       (	self  :  Optional[Any],			_snake_case  :  Any,			_snake_case  :  Dict )       ->str:
            # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
            # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
            with open(_snake_case,			encoding='utf-8' ) as f:
                   for key, row in enumerate(_snake_case ):
                          snake_case__							:     List[str]            =     json.loads(_snake_case )
                          if self.config.name == "first_domain":
                                 # Yields examples as (key, example) tuples
                                 yield key, {
                                     "sentence": data["sentence"],
                                     "option1": data["option1"],
                                     "answer": "" if split == "test" else data["answer"],
                                 }
                          else:
                                 yield key, {
                                     "sentence": data["sentence"],
                                     "option2": data["option2"],
                                     "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
                                 }











import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
 from transformers import WhisperFeatureExtractor, WhisperProcessor


a_       :List[Any]				  =				50_358
a_       :int				  =				50_362




@require_torch
@require_torchaudio
@require_sentencepiece
class  snake_case__						(	unittest.TestCase					):






     """simple docstring"""


     def        lowercase_       (	self  :  Dict )       ->List[Any]:
            snake_case__							:     int            =     'openai/whisper-small.en'
            snake_case__							:     List[Any]            =     tempfile.mkdtemp()






     def        lowercase_       (	self  :  Any,			**_snake_case  :  List[str] )       ->Union[str, Any]:
            return WhisperTokenizer.from_pretrained(self.checkpoint,			**_snake_case )






     def        lowercase_       (	self  :  Union[str, Any],			**_snake_case  :  int )       ->Optional[int]:
            return WhisperFeatureExtractor.from_pretrained(self.checkpoint,			**_snake_case )






     def        lowercase_       (	self  :  Any )       ->Union[str, Any]:
            shutil.rmtree(self.tmpdirname )






     def        lowercase_       (	self  :  Dict )       ->Tuple:
            snake_case__							:     Dict            =     self.get_tokenizer()
            snake_case__							:     List[str]            =     self.get_feature_extractor()

            snake_case__							:     List[Any]            =     WhisperProcessor(tokenizer=_snake_case,			feature_extractor=_snake_case )

            processor.save_pretrained(self.tmpdirname )
            snake_case__							:     Dict            =     WhisperProcessor.from_pretrained(self.tmpdirname )

            self.assertEqual(processor.tokenizer.get_vocab(),			tokenizer.get_vocab() )
            self.assertIsInstance(processor.tokenizer,			_snake_case )

            self.assertEqual(processor.feature_extractor.to_json_string(),			feature_extractor.to_json_string() )
            self.assertIsInstance(processor.feature_extractor,			_snake_case )






     def        lowercase_       (	self  :  List[Any] )       ->str:
            snake_case__							:     str            =     WhisperProcessor(tokenizer=self.get_tokenizer(),			feature_extractor=self.get_feature_extractor() )
            processor.save_pretrained(self.tmpdirname )

            snake_case__							:     Dict            =     self.get_tokenizer(bos_token='(BOS)',			eos_token='(EOS)' )
            snake_case__							:     List[str]            =     self.get_feature_extractor(do_normalize=_snake_case,			padding_value=1.0 )

            snake_case__							:     List[Any]            =     WhisperProcessor.from_pretrained(
                self.tmpdirname,			bos_token='(BOS)',			eos_token='(EOS)',			do_normalize=_snake_case,			padding_value=1.0 )

            self.assertEqual(processor.tokenizer.get_vocab(),			tokenizer_add_kwargs.get_vocab() )
            self.assertIsInstance(processor.tokenizer,			_snake_case )

            self.assertEqual(processor.feature_extractor.to_json_string(),			feature_extractor_add_kwargs.to_json_string() )
            self.assertIsInstance(processor.feature_extractor,			_snake_case )






     def        lowercase_       (	self  :  Union[str, Any] )       ->Dict:
            snake_case__							:     Dict            =     self.get_feature_extractor()
            snake_case__							:     Dict            =     self.get_tokenizer()

            snake_case__							:     List[Any]            =     WhisperProcessor(tokenizer=_snake_case,			feature_extractor=_snake_case )

            snake_case__							:     int            =     floats_list((3, 1_0_0_0) )

            snake_case__							:     Optional[int]            =     feature_extractor(_snake_case,			return_tensors='np' )
            snake_case__							:     Optional[int]            =     processor(_snake_case,			return_tensors='np' )

            for key in input_feat_extract.keys():
                   self.assertAlmostEqual(input_feat_extract[key].sum(),			input_processor[key].sum(),			delta=1e-2 )






     def        lowercase_       (	self  :  Dict )       ->Any:
            snake_case__							:     Tuple            =     self.get_feature_extractor()
            snake_case__							:     Optional[int]            =     self.get_tokenizer()

            snake_case__							:     Optional[int]            =     WhisperProcessor(tokenizer=_snake_case,			feature_extractor=_snake_case )

            snake_case__							:     Any            =     'This is a test string'

            snake_case__							:     Dict            =     processor(text=_snake_case )

            snake_case__							:     List[str]            =     tokenizer(_snake_case )

            for key in encoded_tok.keys():
                   self.assertListEqual(encoded_tok[key],			encoded_processor[key] )






     def        lowercase_       (	self  :  Dict )       ->Dict:
            snake_case__							:     Optional[Any]            =     self.get_feature_extractor()
            snake_case__							:     Optional[int]            =     self.get_tokenizer()

            snake_case__							:     Optional[int]            =     WhisperProcessor(tokenizer=_snake_case,			feature_extractor=_snake_case )

            snake_case__							:     Union[str, Any]            =     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

            snake_case__							:     List[str]            =     processor.batch_decode(_snake_case )
            snake_case__							:     Dict            =     tokenizer.batch_decode(_snake_case )

            self.assertListEqual(_snake_case,			_snake_case )






     def        lowercase_       (	self  :  Union[str, Any] )       ->Optional[Any]:
            snake_case__							:     str            =     self.get_feature_extractor()
            snake_case__							:     Dict            =     self.get_tokenizer()

            snake_case__							:     Any            =     WhisperProcessor(tokenizer=_snake_case,			feature_extractor=_snake_case )

            self.assertListEqual(
                processor.model_input_names,			feature_extractor.model_input_names,			msg='`processor` and `feature_extractor` model input names do not match',			)






     def        lowercase_       (	self  :  int )       ->Dict:
            snake_case__							:     Tuple            =     self.get_feature_extractor()
            snake_case__							:     Tuple            =     self.get_tokenizer()

            snake_case__							:     Optional[int]            =     WhisperProcessor(tokenizer=_snake_case,			feature_extractor=_snake_case )
            snake_case__							:     Any            =     processor.get_decoder_prompt_ids(task='transcribe',			no_timestamps=_snake_case )

            self.assertIsInstance(_snake_case,			_snake_case )
            for ids in forced_decoder_ids:
                   self.assertIsInstance(_snake_case,			(list, tuple) )

            snake_case__							:     List[str]            =     [TRANSCRIBE, NOTIMESTAMPS]
            self.assertListEqual([ids[-1] for ids in forced_decoder_ids],			_snake_case )






     def        lowercase_       (	self  :  Optional[int] )       ->int:
            snake_case__							:     List[Any]            =     WhisperProcessor(tokenizer=self.get_tokenizer(),			feature_extractor=self.get_feature_extractor() )
            snake_case__							:     Optional[Any]            =     processor.get_prompt_ids('Mr. Quilter' )
            snake_case__							:     List[Any]            =     processor.tokenizer.decode(_snake_case )

            self.assertListEqual(prompt_ids.tolist(),			[5_0_3_6_0, 1_7_7_0, 1_3, 2_2_6_4, 3_4_6, 3_5_3] )
            self.assertEqual(_snake_case,			'<|startofprev|> Mr. Quilter' )






     def        lowercase_       (	self  :  Tuple )       ->Tuple:
            snake_case__							:     int            =     WhisperProcessor(tokenizer=self.get_tokenizer(),			feature_extractor=self.get_feature_extractor() )
            snake_case__							:     List[str]            =     processor.get_prompt_ids('' )
            snake_case__							:     List[Any]            =     processor.tokenizer.decode(_snake_case )

            self.assertListEqual(prompt_ids.tolist(),			[5_0_3_6_0, 2_2_0] )
            self.assertEqual(_snake_case,			'<|startofprev|> ' )






     def        lowercase_       (	self  :  Tuple )       ->str:
            snake_case__							:     Any            =     WhisperProcessor(tokenizer=self.get_tokenizer(),			feature_extractor=self.get_feature_extractor() )

            def _test_prompt_error_raised_helper(_snake_case  :  str,			_snake_case  :  int ):
                   with pytest.raises(_snake_case ) as excinfo:
                          processor.get_prompt_ids(_snake_case )
                   snake_case__							:     Dict            =     F'''Encountered text in the prompt corresponding to disallowed special token: {special_token}.'''
                   self.assertEqual(_snake_case,			str(excinfo.value ) )

            _test_prompt_error_raised_helper('<|startofprev|> test',			'<|startofprev|>' )
            _test_prompt_error_raised_helper('test <|notimestamps|>',			'<|notimestamps|>' )
            _test_prompt_error_raised_helper('test <|zh|> test <|transcribe|>',			'<|zh|>' )






