

import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
							from transformers import WhisperFeatureExtractor, WhisperProcessor


A_						:     Union[str, Any]			  =				50358
A_						:     int			  =				50362
@require_torch
@require_torchaudio
@require_sentencepiece
class    A_				( unittest.TestCase  ):


				'''simple docstring'''






				def        lowerCAmelCase_  (self					)       -> Optional[Any]:
								__UpperCAmelCase			   =     '''openai/whisper-small.en'''
								__UpperCAmelCase			   =     tempfile.mkdtemp()


				def        lowerCAmelCase_  (self       , **lowercase__					)       -> Optional[int]:
								return WhisperTokenizer.from_pretrained(self.checkpoint       , **_lowerCamelCase					)


				def        lowerCAmelCase_  (self       , **lowercase__					)       -> str:
								return WhisperFeatureExtractor.from_pretrained(self.checkpoint       , **_lowerCamelCase					)


				def        lowerCAmelCase_  (self					)       -> Union[str, Any]:
								shutil.rmtree(self.tmpdirname					)


				def        lowerCAmelCase_  (self					)       -> int:
								__UpperCAmelCase			   =     self.get_tokenizer()
								__UpperCAmelCase			   =     self.get_feature_extractor()

								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=_lowerCamelCase       , feature_extractor=_lowerCamelCase					)

								processor.save_pretrained(self.tmpdirname					)
								__UpperCAmelCase			   =     WhisperProcessor.from_pretrained(self.tmpdirname					)

								self.assertEqual(processor.tokenizer.get_vocab()       , tokenizer.get_vocab()					)
								self.assertIsInstance(processor.tokenizer       , _lowerCamelCase					)

								self.assertEqual(processor.feature_extractor.to_json_string()       , feature_extractor.to_json_string()					)
								self.assertIsInstance(processor.feature_extractor       , _lowerCamelCase					)


				def        lowerCAmelCase_  (self					)       -> Union[str, Any]:
								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=self.get_tokenizer()       , feature_extractor=self.get_feature_extractor()					)
								processor.save_pretrained(self.tmpdirname					)

								__UpperCAmelCase			   =     self.get_tokenizer(bos_token='''(BOS)'''       , eos_token='''(EOS)'''					)
								__UpperCAmelCase			   =     self.get_feature_extractor(do_normalize=_lowerCamelCase       , padding_value=1.0					)

								__UpperCAmelCase			   =     WhisperProcessor.from_pretrained(
								    self.tmpdirname       , bos_token='''(BOS)'''       , eos_token='''(EOS)'''       , do_normalize=_lowerCamelCase       , padding_value=1.0					)

								self.assertEqual(processor.tokenizer.get_vocab()       , tokenizer_add_kwargs.get_vocab()					)
								self.assertIsInstance(processor.tokenizer       , _lowerCamelCase					)

								self.assertEqual(processor.feature_extractor.to_json_string()       , feature_extractor_add_kwargs.to_json_string()					)
								self.assertIsInstance(processor.feature_extractor       , _lowerCamelCase					)


				def        lowerCAmelCase_  (self					)       -> Union[str, Any]:
								__UpperCAmelCase			   =     self.get_feature_extractor()
								__UpperCAmelCase			   =     self.get_tokenizer()

								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=_lowerCamelCase       , feature_extractor=_lowerCamelCase					)

								__UpperCAmelCase			   =     floats_list((3, 1_000)					)

								__UpperCAmelCase			   =     feature_extractor(_lowerCamelCase       , return_tensors='''np'''					)
								__UpperCAmelCase			   =     processor(_lowerCamelCase       , return_tensors='''np'''					)

								for key in input_feat_extract.keys():
												self.assertAlmostEqual(input_feat_extract[key].sum()       , input_processor[key].sum()       , delta=1E-2					)


				def        lowerCAmelCase_  (self					)       -> List[Any]:
								__UpperCAmelCase			   =     self.get_feature_extractor()
								__UpperCAmelCase			   =     self.get_tokenizer()

								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=_lowerCamelCase       , feature_extractor=_lowerCamelCase					)

								__UpperCAmelCase			   =     '''This is a test string'''

								__UpperCAmelCase			   =     processor(text=_lowerCamelCase					)

								__UpperCAmelCase			   =     tokenizer(_lowerCamelCase					)

								for key in encoded_tok.keys():
												self.assertListEqual(encoded_tok[key]       , encoded_processor[key]					)


				def        lowerCAmelCase_  (self					)       -> List[Any]:
								__UpperCAmelCase			   =     self.get_feature_extractor()
								__UpperCAmelCase			   =     self.get_tokenizer()

								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=_lowerCamelCase       , feature_extractor=_lowerCamelCase					)

								__UpperCAmelCase			   =     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								__UpperCAmelCase			   =     processor.batch_decode(_lowerCamelCase					)
								__UpperCAmelCase			   =     tokenizer.batch_decode(_lowerCamelCase					)

								self.assertListEqual(_lowerCamelCase       , _lowerCamelCase					)


				def        lowerCAmelCase_  (self					)       -> List[str]:
								__UpperCAmelCase			   =     self.get_feature_extractor()
								__UpperCAmelCase			   =     self.get_tokenizer()

								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=_lowerCamelCase       , feature_extractor=_lowerCamelCase					)

								self.assertListEqual(
								    processor.model_input_names       , feature_extractor.model_input_names       , msg='''`processor` and `feature_extractor` model input names do not match'''       , )


				def        lowerCAmelCase_  (self					)       -> List[str]:
								__UpperCAmelCase			   =     self.get_feature_extractor()
								__UpperCAmelCase			   =     self.get_tokenizer()

								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=_lowerCamelCase       , feature_extractor=_lowerCamelCase					)
								__UpperCAmelCase			   =     processor.get_decoder_prompt_ids(task='''transcribe'''       , no_timestamps=_lowerCamelCase					)

								self.assertIsInstance(_lowerCamelCase       , _lowerCamelCase					)
								for ids in forced_decoder_ids:
												self.assertIsInstance(_lowerCamelCase       , (list, tuple)					)

								__UpperCAmelCase			   =     [TRANSCRIBE, NOTIMESTAMPS]
								self.assertListEqual([ids[-1] for ids in forced_decoder_ids]       , _lowerCamelCase					)


				def        lowerCAmelCase_  (self					)       -> int:
								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=self.get_tokenizer()       , feature_extractor=self.get_feature_extractor()					)
								__UpperCAmelCase			   =     processor.get_prompt_ids('''Mr. Quilter'''					)
								__UpperCAmelCase			   =     processor.tokenizer.decode(_lowerCamelCase					)

								self.assertListEqual(prompt_ids.tolist()       , [50_360, 1_770, 13, 2_264, 346, 353]					)
								self.assertEqual(_lowerCamelCase       , '''<|startofprev|> Mr. Quilter'''					)


				def        lowerCAmelCase_  (self					)       -> Tuple:
								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=self.get_tokenizer()       , feature_extractor=self.get_feature_extractor()					)
								__UpperCAmelCase			   =     processor.get_prompt_ids(''''''					)
								__UpperCAmelCase			   =     processor.tokenizer.decode(_lowerCamelCase					)

								self.assertListEqual(prompt_ids.tolist()       , [50_360, 220]					)
								self.assertEqual(_lowerCamelCase       , '''<|startofprev|> '''					)






				def        lowerCAmelCase_  (self					)       -> int:
								__UpperCAmelCase			   =     WhisperProcessor(tokenizer=self.get_tokenizer()       , feature_extractor=self.get_feature_extractor()					)

								def _test_prompt_error_raised_helper(lowercase__       , lowercase__					):
												with pytest.raises(_lowerCamelCase					) as excinfo:
																processor.get_prompt_ids(_lowerCamelCase					)
												__UpperCAmelCase			   =     F'''Encountered text in the prompt corresponding to disallowed special token: {special_token}.'''
												self.assertEqual(_lowerCamelCase       , str(excinfo.value					)					)

								_test_prompt_error_raised_helper('''<|startofprev|> test'''       , '''<|startofprev|>'''					)
								_test_prompt_error_raised_helper('''test <|notimestamps|>'''       , '''<|notimestamps|>'''					)
								_test_prompt_error_raised_helper('''test <|zh|> test <|transcribe|>'''       , '''<|zh|>'''					)







from dataclasses import dataclass
from typing import Callable, Optional

import datasets
@dataclass
class    A_				( datasets.BuilderConfig  ):


    '''simple docstring'''






    a__       							=    None
    a__       							=    None
    a__       							=    None


    def        lowerCAmelCase_  (self					)       -> Dict:
        assert self.generator is not None, "generator must be specified"

        if self.gen_kwargs is None:
            __UpperCAmelCase			   =     {}






class    A_				( datasets.GeneratorBasedBuilder  ):


    '''simple docstring'''






    a__       							=    GeneratorConfig


    def        lowerCAmelCase_  (self					)       -> Dict:
        return datasets.DatasetInfo(features=self.config.features					)


    def        lowerCAmelCase_  (self       , lowercase__					)       -> List[str]:
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN       , gen_kwargs=self.config.gen_kwargs					)]






    def        lowerCAmelCase_  (self       , **lowercase__					)       -> str:
        for idx, ex in enumerate(self.config.generator(**lowercase__					)					):
            yield idx, ex




