




import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
  from transformers import WhisperFeatureExtractor, WhisperProcessor


A							:				Optional[int]			     =	50_358
A							:				List[str]			     =	50_362






@require_torch
@require_torchaudio
@require_sentencepiece
class 	SCREAMING_SNAKE_CASE(    unittest.TestCase					):

       def    snake_case__      (   self					)   -> str:

           """simple docstring"""



           __lowercase          =						"openai/whisper-small.en"
           __lowercase          =						tempfile.mkdtemp()

       def    snake_case__      (   self ,		**lowerCamelCase__					)   -> List[Any]:

           """simple docstring"""



           return WhisperTokenizer.from_pretrained(self.checkpoint ,		**SCREAMING_SNAKE_CASE__					)

       def    snake_case__      (   self ,		**lowerCamelCase__					)   -> Tuple:

           """simple docstring"""



           return WhisperFeatureExtractor.from_pretrained(self.checkpoint ,		**SCREAMING_SNAKE_CASE__					)

       def    snake_case__      (   self					)   -> Dict:

           """simple docstring"""



           shutil.rmtree(self.tmpdirname					)

       def    snake_case__      (   self					)   -> Dict:

           """simple docstring"""



           __lowercase          =						self.get_tokenizer()
           __lowercase          =						self.get_feature_extractor()

           __lowercase          =						WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__ ,		feature_extractor=SCREAMING_SNAKE_CASE__					)

           processor.save_pretrained(self.tmpdirname					)
           __lowercase          =						WhisperProcessor.from_pretrained(self.tmpdirname					)

           self.assertEqual(processor.tokenizer.get_vocab() ,		tokenizer.get_vocab()					)
           self.assertIsInstance(processor.tokenizer ,		SCREAMING_SNAKE_CASE__					)

           self.assertEqual(processor.feature_extractor.to_json_string() ,		feature_extractor.to_json_string()					)
           self.assertIsInstance(processor.feature_extractor ,		SCREAMING_SNAKE_CASE__					)

       def    snake_case__      (   self					)   -> int:

           """simple docstring"""



           __lowercase          =						WhisperProcessor(tokenizer=self.get_tokenizer() ,		feature_extractor=self.get_feature_extractor()					)
           processor.save_pretrained(self.tmpdirname					)

           __lowercase          =						self.get_tokenizer(bos_token="""(BOS)""" ,		eos_token="""(EOS)"""					)
           __lowercase          =						self.get_feature_extractor(do_normalize=SCREAMING_SNAKE_CASE__ ,		padding_value=1.0					)

           __lowercase          =						WhisperProcessor.from_pretrained(
               self.tmpdirname ,		bos_token="""(BOS)""" ,		eos_token="""(EOS)""" ,		do_normalize=SCREAMING_SNAKE_CASE__ ,		padding_value=1.0					)

           self.assertEqual(processor.tokenizer.get_vocab() ,		tokenizer_add_kwargs.get_vocab()					)
           self.assertIsInstance(processor.tokenizer ,		SCREAMING_SNAKE_CASE__					)

           self.assertEqual(processor.feature_extractor.to_json_string() ,		feature_extractor_add_kwargs.to_json_string()					)
           self.assertIsInstance(processor.feature_extractor ,		SCREAMING_SNAKE_CASE__					)

       def    snake_case__      (   self					)   -> Union[str, Any]:

           """simple docstring"""



           __lowercase          =						self.get_feature_extractor()
           __lowercase          =						self.get_tokenizer()

           __lowercase          =						WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__ ,		feature_extractor=SCREAMING_SNAKE_CASE__					)

           __lowercase          =						floats_list((3, 1000)					)

           __lowercase          =						feature_extractor(SCREAMING_SNAKE_CASE__ ,		return_tensors="""np"""					)
           __lowercase          =						processor(SCREAMING_SNAKE_CASE__ ,		return_tensors="""np"""					)

           for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum() ,		input_processor[key].sum() ,		delta=1E-2					)

       def    snake_case__      (   self					)   -> Dict:

           """simple docstring"""



           __lowercase          =						self.get_feature_extractor()
           __lowercase          =						self.get_tokenizer()

           __lowercase          =						WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__ ,		feature_extractor=SCREAMING_SNAKE_CASE__					)

           __lowercase          =						"This is a test string"

           __lowercase          =						processor(text=SCREAMING_SNAKE_CASE__					)

           __lowercase          =						tokenizer(SCREAMING_SNAKE_CASE__					)

           for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key] ,		encoded_processor[key]					)

       def    snake_case__      (   self					)   -> int:

           """simple docstring"""



           __lowercase          =						self.get_feature_extractor()
           __lowercase          =						self.get_tokenizer()

           __lowercase          =						WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__ ,		feature_extractor=SCREAMING_SNAKE_CASE__					)

           __lowercase          =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           __lowercase          =						processor.batch_decode(SCREAMING_SNAKE_CASE__					)
           __lowercase          =						tokenizer.batch_decode(SCREAMING_SNAKE_CASE__					)

           self.assertListEqual(SCREAMING_SNAKE_CASE__ ,		SCREAMING_SNAKE_CASE__					)

       def    snake_case__      (   self					)   -> Optional[Any]:

           """simple docstring"""



           __lowercase          =						self.get_feature_extractor()
           __lowercase          =						self.get_tokenizer()

           __lowercase          =						WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__ ,		feature_extractor=SCREAMING_SNAKE_CASE__					)

           self.assertListEqual(
               processor.model_input_names ,		feature_extractor.model_input_names ,		msg="""`processor` and `feature_extractor` model input names do not match""" ,		)

       def    snake_case__      (   self					)   -> List[str]:

           """simple docstring"""



           __lowercase          =						self.get_feature_extractor()
           __lowercase          =						self.get_tokenizer()

           __lowercase          =						WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__ ,		feature_extractor=SCREAMING_SNAKE_CASE__					)
           __lowercase          =						processor.get_decoder_prompt_ids(task="""transcribe""" ,		no_timestamps=SCREAMING_SNAKE_CASE__					)

           self.assertIsInstance(SCREAMING_SNAKE_CASE__ ,		SCREAMING_SNAKE_CASE__					)
           for ids in forced_decoder_ids:
               self.assertIsInstance(SCREAMING_SNAKE_CASE__ ,		(list, tuple)					)

           __lowercase          =						[TRANSCRIBE, NOTIMESTAMPS]
           self.assertListEqual([ids[-1] for ids in forced_decoder_ids] ,		SCREAMING_SNAKE_CASE__					)

       def    snake_case__      (   self					)   -> Optional[int]:

           """simple docstring"""



           __lowercase          =						WhisperProcessor(tokenizer=self.get_tokenizer() ,		feature_extractor=self.get_feature_extractor()					)
           __lowercase          =						processor.get_prompt_ids("""Mr. Quilter"""					)
           __lowercase          =						processor.tokenizer.decode(SCREAMING_SNAKE_CASE__					)

           self.assertListEqual(prompt_ids.tolist() ,		[5_0360, 1770, 13, 2264, 346, 353]					)
           self.assertEqual(SCREAMING_SNAKE_CASE__ ,		"""<|startofprev|> Mr. Quilter"""					)

       def    snake_case__      (   self					)   -> Optional[Any]:

           """simple docstring"""



           __lowercase          =						WhisperProcessor(tokenizer=self.get_tokenizer() ,		feature_extractor=self.get_feature_extractor()					)
           __lowercase          =						processor.get_prompt_ids(""""""					)
           __lowercase          =						processor.tokenizer.decode(SCREAMING_SNAKE_CASE__					)

           self.assertListEqual(prompt_ids.tolist() ,		[5_0360, 220]					)
           self.assertEqual(SCREAMING_SNAKE_CASE__ ,		"""<|startofprev|> """					)




       def    snake_case__      (   self					)   -> Dict:

           """simple docstring"""



           __lowercase          =						WhisperProcessor(tokenizer=self.get_tokenizer() ,		feature_extractor=self.get_feature_extractor()					)

           def _test_prompt_error_raised_helper(lowerCamelCase__ ,		lowerCamelCase__					):
               with pytest.raises(SCREAMING_SNAKE_CASE__					) as excinfo:
                   processor.get_prompt_ids(SCREAMING_SNAKE_CASE__					)
               __lowercase          =						F'Encountered text in the prompt corresponding to disallowed special token: {special_token}.'
               self.assertEqual(SCREAMING_SNAKE_CASE__ ,		str(excinfo.value					)					)

           _test_prompt_error_raised_helper("""<|startofprev|> test""" ,		"""<|startofprev|>"""					)
           _test_prompt_error_raised_helper("""test <|notimestamps|>""" ,		"""<|notimestamps|>"""					)
           _test_prompt_error_raised_helper("""test <|zh|> test <|transcribe|>""" ,		"""<|zh|>"""					)










'''simple docstring'''
import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir






@dataclass
class 	SCREAMING_SNAKE_CASE:
       snake_case_			:					str






class 	SCREAMING_SNAKE_CASE(    unittest.TestCase					):

       def    snake_case__      (   self					)   -> Optional[int]:

           """simple docstring"""



           __lowercase          =						get_tests_dir("""fixtures/test_sentencepiece.model"""					)
           __lowercase          =						get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model"""					)

           __lowercase          =						FakeOriginalTokenizer(vocab_file=lowerCamelCase__					)

           with warnings.catch_warnings(record=lowerCamelCase__					) as w:
               __lowercase          =						SpmConverter(lowerCamelCase__					)
           self.assertEqual(len(lowerCamelCase__					) ,		0					)

           __lowercase          =						FakeOriginalTokenizer(vocab_file=lowerCamelCase__					)

           with warnings.catch_warnings(record=lowerCamelCase__					) as w:
               __lowercase          =						SpmConverter(lowerCamelCase__					)
           self.assertEqual(len(lowerCamelCase__					) ,		1					)

           self.assertIn(
               """The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option"""
               """ which is not implemented in the fast tokenizers.""" ,		str(w[0].message					) ,		)




