



"""simple docstring"""
import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil





@pytest.fixture
def        __magic_name__			(							lowercase	):
   SCREAMING_SNAKE_CASE_:	Optional[Any]            =tmp_path / """text.txt"""
   SCREAMING_SNAKE_CASE_:	List[str]            =textwrap.dedent(
       """\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        """	)
   with open(lowercase							,     """w"""							,     encoding="""utf-8"""	) as f:
      f.write(lowercase	)
   return str(lowercase	)





@pytest.fixture
def        __magic_name__			(							lowercase							,     lowercase	):
   SCREAMING_SNAKE_CASE_:	Dict            =tmp_path / """text_with_image.txt"""
   with open(lowercase							,     """w"""							,     encoding="""utf-8"""	) as f:
      f.write(lowercase	)
   return str(lowercase	)





@pytest.mark.parametrize("""keep_linebreaks"""							,     [True, False]	)
def        __magic_name__			(							lowercase							,     lowercase	):
   with open(lowercase							,     encoding="""utf-8"""	) as f:
      SCREAMING_SNAKE_CASE_:	Tuple            =f.read().splitlines(keepends=lowercase	)
   SCREAMING_SNAKE_CASE_:	List[str]            =Text(keep_linebreaks=lowercase							,     encoding="""utf-8"""	)
   SCREAMING_SNAKE_CASE_:	Union[str, Any]            =text._generate_tables([[text_file]]	)
   SCREAMING_SNAKE_CASE_:	List[Any]            =pa.concat_tables([table for _, table in generator]	).to_pydict()["""text"""]
   assert generated_content == expected_content





@require_pil
def        __magic_name__			(							lowercase	):
   with open(lowercase							,     encoding="""utf-8"""	) as f:
      SCREAMING_SNAKE_CASE_:	Optional[int]            =f.read().splitlines()[0]
   SCREAMING_SNAKE_CASE_:	Any            =Text(encoding="""utf-8"""							,     features=Features({"""image""": Image()}	)	)
   SCREAMING_SNAKE_CASE_:	Optional[Any]            =text._generate_tables([[text_file_with_image]]	)
   SCREAMING_SNAKE_CASE_:	Union[str, Any]            =pa.concat_tables([table for _, table in generator]	)
   assert pa_table.schema.field("""image"""	).type == Image()()
   SCREAMING_SNAKE_CASE_:	Union[str, Any]            =pa_table.to_pydict()["""image"""]
   assert generated_content == [{"path": image_file, "bytes": None}]





@pytest.mark.parametrize("""sample_by"""							,     ["""line""", """paragraph""", """document"""]	)
def        __magic_name__			(							lowercase							,     lowercase	):
   with open(lowercase							,     encoding="""utf-8"""	) as f:
      SCREAMING_SNAKE_CASE_:	Union[str, Any]            =f.read()
   if sample_by == "line":
      SCREAMING_SNAKE_CASE_:	List[str]            =expected_content.splitlines()
   elif sample_by == "paragraph":
      SCREAMING_SNAKE_CASE_:	Any            =expected_content.split("""\n\n"""	)
   elif sample_by == "document":
      SCREAMING_SNAKE_CASE_:	Any            =[expected_content]
   SCREAMING_SNAKE_CASE_:	str            =Text(sample_by=lowercase							,     encoding="""utf-8"""							,     chunksize=100	)
   SCREAMING_SNAKE_CASE_:	int            =text._generate_tables([[text_file]]	)
   SCREAMING_SNAKE_CASE_:	str            =pa.concat_tables([table for _, table in generator]	).to_pydict()["""text"""]
   assert generated_content == expected_content









"""simple docstring"""
import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
   from transformers import WhisperFeatureExtractor, WhisperProcessor


_UpperCAmelCase       =     5_0_3_5_8
_UpperCAmelCase       =     5_0_3_6_2
@require_torch
@require_torchaudio
@require_sentencepiece
class    a		(					unittest.TestCase    ):



  def 		lowerCamelCase__    (  self	:   Dict						)       ->   Tuple:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	List[Any]            ="""openai/whisper-small.en"""
     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =tempfile.mkdtemp()



  def 		lowerCamelCase__    (  self	:   Any    ,						**lowerCAmelCase	:   str						)       ->   str:


     '''simple docstring'''




     return WhisperTokenizer.from_pretrained(self.checkpoint    ,						**lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   str    ,						**lowerCAmelCase	:   Optional[Any]						)       ->   Optional[Any]:


     '''simple docstring'''




     return WhisperFeatureExtractor.from_pretrained(self.checkpoint    ,						**lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   Tuple						)       ->   Optional[Any]:


     '''simple docstring'''




     shutil.rmtree(self.tmpdirname						)



  def 		lowerCamelCase__    (  self	:   int						)       ->   str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[int]            =self.get_tokenizer()
     SCREAMING_SNAKE_CASE_:	Optional[int]            =self.get_feature_extractor()

     SCREAMING_SNAKE_CASE_:	Optional[Any]            =WhisperProcessor(tokenizer=lowerCAmelCase    ,						feature_extractor=lowerCAmelCase						)

     processor.save_pretrained(self.tmpdirname						)
     SCREAMING_SNAKE_CASE_:	Dict            =WhisperProcessor.from_pretrained(self.tmpdirname						)

     self.assertEqual(processor.tokenizer.get_vocab()    ,						tokenizer.get_vocab()						)
     self.assertIsInstance(processor.tokenizer    ,						lowerCAmelCase						)

     self.assertEqual(processor.feature_extractor.to_json_string()    ,						feature_extractor.to_json_string()						)
     self.assertIsInstance(processor.feature_extractor    ,						lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   str						)       ->   Optional[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =WhisperProcessor(tokenizer=self.get_tokenizer()    ,						feature_extractor=self.get_feature_extractor()						)
     processor.save_pretrained(self.tmpdirname						)

     SCREAMING_SNAKE_CASE_:	int            =self.get_tokenizer(bos_token="""(BOS)"""    ,						eos_token="""(EOS)"""						)
     SCREAMING_SNAKE_CASE_:	Dict            =self.get_feature_extractor(do_normalize=lowerCAmelCase    ,						padding_value=1.0						)

     SCREAMING_SNAKE_CASE_:	Optional[int]            =WhisperProcessor.from_pretrained(
         self.tmpdirname    ,						bos_token="""(BOS)"""    ,						eos_token="""(EOS)"""    ,						do_normalize=lowerCAmelCase    ,						padding_value=1.0						)

     self.assertEqual(processor.tokenizer.get_vocab()    ,						tokenizer_add_kwargs.get_vocab()						)
     self.assertIsInstance(processor.tokenizer    ,						lowerCAmelCase						)

     self.assertEqual(processor.feature_extractor.to_json_string()    ,						feature_extractor_add_kwargs.to_json_string()						)
     self.assertIsInstance(processor.feature_extractor    ,						lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   Tuple						)       ->   Optional[int]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Tuple            =self.get_feature_extractor()
     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =self.get_tokenizer()

     SCREAMING_SNAKE_CASE_:	Dict            =WhisperProcessor(tokenizer=lowerCAmelCase    ,						feature_extractor=lowerCAmelCase						)

     SCREAMING_SNAKE_CASE_:	Dict            =floats_list((3, 1000)						)

     SCREAMING_SNAKE_CASE_:	Optional[Any]            =feature_extractor(lowerCAmelCase    ,						return_tensors="""np"""						)
     SCREAMING_SNAKE_CASE_:	str            =processor(lowerCAmelCase    ,						return_tensors="""np"""						)

     for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()    ,						input_processor[key].sum()    ,						delta=1E-2						)



  def 		lowerCamelCase__    (  self	:   Tuple						)       ->   Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	List[str]            =self.get_feature_extractor()
     SCREAMING_SNAKE_CASE_:	str            =self.get_tokenizer()

     SCREAMING_SNAKE_CASE_:	List[Any]            =WhisperProcessor(tokenizer=lowerCAmelCase    ,						feature_extractor=lowerCAmelCase						)

     SCREAMING_SNAKE_CASE_:	List[str]            ="""This is a test string"""

     SCREAMING_SNAKE_CASE_:	Tuple            =processor(text=lowerCAmelCase						)

     SCREAMING_SNAKE_CASE_:	List[Any]            =tokenizer(lowerCAmelCase						)

     for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]    ,						encoded_processor[key]						)



  def 		lowerCamelCase__    (  self	:   Optional[Any]						)       ->   Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[int]            =self.get_feature_extractor()
     SCREAMING_SNAKE_CASE_:	str            =self.get_tokenizer()

     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =WhisperProcessor(tokenizer=lowerCAmelCase    ,						feature_extractor=lowerCAmelCase						)

     SCREAMING_SNAKE_CASE_:	List[str]            =[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     SCREAMING_SNAKE_CASE_:	Optional[int]            =processor.batch_decode(lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	Dict            =tokenizer.batch_decode(lowerCAmelCase						)

     self.assertListEqual(lowerCAmelCase    ,						lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   Union[str, Any]						)       ->   Dict:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	str            =self.get_feature_extractor()
     SCREAMING_SNAKE_CASE_:	List[Any]            =self.get_tokenizer()

     SCREAMING_SNAKE_CASE_:	Optional[Any]            =WhisperProcessor(tokenizer=lowerCAmelCase    ,						feature_extractor=lowerCAmelCase						)

     self.assertListEqual(
         processor.model_input_names    ,						feature_extractor.model_input_names    ,						msg="""`processor` and `feature_extractor` model input names do not match"""    ,						)



  def 		lowerCamelCase__    (  self	:   Optional[int]						)       ->   Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[Any]            =self.get_feature_extractor()
     SCREAMING_SNAKE_CASE_:	Any            =self.get_tokenizer()

     SCREAMING_SNAKE_CASE_:	List[Any]            =WhisperProcessor(tokenizer=lowerCAmelCase    ,						feature_extractor=lowerCAmelCase						)
     SCREAMING_SNAKE_CASE_:	Any            =processor.get_decoder_prompt_ids(task="""transcribe"""    ,						no_timestamps=lowerCAmelCase						)

     self.assertIsInstance(lowerCAmelCase    ,						lowerCAmelCase						)
     for ids in forced_decoder_ids:
        self.assertIsInstance(lowerCAmelCase    ,						(list, tuple)						)

     SCREAMING_SNAKE_CASE_:	Tuple            =[TRANSCRIBE, NOTIMESTAMPS]
     self.assertListEqual([ids[-1] for ids in forced_decoder_ids]    ,						lowerCAmelCase						)



  def 		lowerCamelCase__    (  self	:   List[str]						)       ->   List[str]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =WhisperProcessor(tokenizer=self.get_tokenizer()    ,						feature_extractor=self.get_feature_extractor()						)
     SCREAMING_SNAKE_CASE_:	Optional[int]            =processor.get_prompt_ids("""Mr. Quilter"""						)
     SCREAMING_SNAKE_CASE_:	List[str]            =processor.tokenizer.decode(lowerCAmelCase						)

     self.assertListEqual(prompt_ids.tolist()    ,						[5_0360, 1770, 13, 2264, 346, 353]						)
     self.assertEqual(lowerCAmelCase    ,						"""<|startofprev|> Mr. Quilter"""						)



  def 		lowerCamelCase__    (  self	:   Dict						)       ->   Tuple:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Optional[Any]            =WhisperProcessor(tokenizer=self.get_tokenizer()    ,						feature_extractor=self.get_feature_extractor()						)
     SCREAMING_SNAKE_CASE_:	int            =processor.get_prompt_ids(""""""						)
     SCREAMING_SNAKE_CASE_:	Union[str, Any]            =processor.tokenizer.decode(lowerCAmelCase						)

     self.assertListEqual(prompt_ids.tolist()    ,						[5_0360, 220]						)
     self.assertEqual(lowerCAmelCase    ,						"""<|startofprev|> """						)





  def 		lowerCamelCase__    (  self	:   Optional[int]						)       ->   Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:	Dict            =WhisperProcessor(tokenizer=self.get_tokenizer()    ,						feature_extractor=self.get_feature_extractor()						)

     def _test_prompt_error_raised_helper(lowerCAmelCase	:   List[str]    ,						lowerCAmelCase	:   int						):
        with pytest.raises(lowerCAmelCase						) as excinfo:
           processor.get_prompt_ids(lowerCAmelCase						)
        SCREAMING_SNAKE_CASE_:	List[Any]            =f'''Encountered text in the prompt corresponding to disallowed special token: {special_token}.'''
        self.assertEqual(lowerCAmelCase    ,						str(excinfo.value						)						)

     _test_prompt_error_raised_helper("""<|startofprev|> test"""    ,						"""<|startofprev|>"""						)
     _test_prompt_error_raised_helper("""test <|notimestamps|>"""    ,						"""<|notimestamps|>"""						)
     _test_prompt_error_raised_helper("""test <|zh|> test <|transcribe|>"""    ,						"""<|zh|>"""						)



