




'''simple docstring'''






import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
				from transformers import WhisperFeatureExtractor, WhisperProcessor


_lowerCAmelCase				     = 50358
_lowerCAmelCase				     = 50362







@require_torch
@require_torchaudio
@require_sentencepiece
class  A		(			unittest.TestCase		):




				'''simple docstring'''





				def 	a_	(self      )     ->	List[Any]:
							__UpperCamelCase				:      int									=       'openai/whisper-small.en'
							__UpperCamelCase				:      List[Any]									=       tempfile.mkdtemp()
				def 	a_	(self			,  **_UpperCAmelCase      )     ->	Union[str, Any]:
							return WhisperTokenizer.from_pretrained(self.checkpoint			,  **_snake_case      )
				def 	a_	(self			,  **_UpperCAmelCase      )     ->	Optional[int]:
							return WhisperFeatureExtractor.from_pretrained(self.checkpoint			,  **_snake_case      )
				def 	a_	(self      )     ->	Union[str, Any]:
							shutil.rmtree(self.tmpdirname      )
				def 	a_	(self      )     ->	Tuple:
							__UpperCamelCase				:      Dict									=       self.get_tokenizer()
							__UpperCamelCase				:      List[str]									=       self.get_feature_extractor()

							__UpperCamelCase				:      List[Any]									=       WhisperProcessor(tokenizer=_snake_case			,  feature_extractor=_snake_case      )

							processor.save_pretrained(self.tmpdirname      )
							__UpperCamelCase				:      Dict									=       WhisperProcessor.from_pretrained(self.tmpdirname      )

							self.assertEqual(processor.tokenizer.get_vocab()			,  tokenizer.get_vocab()      )
							self.assertIsInstance(processor.tokenizer			,  _snake_case      )

							self.assertEqual(processor.feature_extractor.to_json_string()			,  feature_extractor.to_json_string()      )
							self.assertIsInstance(processor.feature_extractor			,  _snake_case      )
				def 	a_	(self      )     ->	str:
							__UpperCamelCase				:      str									=       WhisperProcessor(tokenizer=self.get_tokenizer()			,  feature_extractor=self.get_feature_extractor()      )
							processor.save_pretrained(self.tmpdirname      )

							__UpperCamelCase				:      Dict									=       self.get_tokenizer(bos_token="(BOS)"			,  eos_token="(EOS)"      )
							__UpperCamelCase				:      List[str]									=       self.get_feature_extractor(do_normalize=_snake_case			,  padding_value=1.0      )

							__UpperCamelCase				:      List[Any]									=       WhisperProcessor.from_pretrained(
							    self.tmpdirname			,  bos_token="(BOS)"			,  eos_token="(EOS)"			,  do_normalize=_snake_case			,  padding_value=1.0      )

							self.assertEqual(processor.tokenizer.get_vocab()			,  tokenizer_add_kwargs.get_vocab()      )
							self.assertIsInstance(processor.tokenizer			,  _snake_case      )

							self.assertEqual(processor.feature_extractor.to_json_string()			,  feature_extractor_add_kwargs.to_json_string()      )
							self.assertIsInstance(processor.feature_extractor			,  _snake_case      )
				def 	a_	(self      )     ->	Dict:
							__UpperCamelCase				:      Dict									=       self.get_feature_extractor()
							__UpperCamelCase				:      Dict									=       self.get_tokenizer()

							__UpperCamelCase				:      List[Any]									=       WhisperProcessor(tokenizer=_snake_case			,  feature_extractor=_snake_case      )

							__UpperCamelCase				:      int									=       floats_list((3, 1_0_0_0)      )

							__UpperCamelCase				:      Optional[int]									=       feature_extractor(_snake_case			,  return_tensors="np"      )
							__UpperCamelCase				:      Optional[int]									=       processor(_snake_case			,  return_tensors="np"      )

							for key in input_feat_extract.keys():
										self.assertAlmostEqual(input_feat_extract[key].sum()			,  input_processor[key].sum()			,  delta=1E-2      )
				def 	a_	(self      )     ->	Any:
							__UpperCamelCase				:      Tuple									=       self.get_feature_extractor()
							__UpperCamelCase				:      Optional[int]									=       self.get_tokenizer()

							__UpperCamelCase				:      Optional[int]									=       WhisperProcessor(tokenizer=_snake_case			,  feature_extractor=_snake_case      )

							__UpperCamelCase				:      Any									=       'This is a test string'

							__UpperCamelCase				:      Dict									=       processor(text=_snake_case      )

							__UpperCamelCase				:      List[str]									=       tokenizer(_snake_case      )

							for key in encoded_tok.keys():
										self.assertListEqual(encoded_tok[key]			,  encoded_processor[key]      )
				def 	a_	(self      )     ->	Dict:
							__UpperCamelCase				:      Optional[Any]									=       self.get_feature_extractor()
							__UpperCamelCase				:      Optional[int]									=       self.get_tokenizer()

							__UpperCamelCase				:      Optional[int]									=       WhisperProcessor(tokenizer=_snake_case			,  feature_extractor=_snake_case      )

							__UpperCamelCase				:      Union[str, Any]									=       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

							__UpperCamelCase				:      List[str]									=       processor.batch_decode(_snake_case      )
							__UpperCamelCase				:      Dict									=       tokenizer.batch_decode(_snake_case      )

							self.assertListEqual(_snake_case			,  _snake_case      )
				def 	a_	(self      )     ->	Optional[Any]:
							__UpperCamelCase				:      str									=       self.get_feature_extractor()
							__UpperCamelCase				:      Dict									=       self.get_tokenizer()

							__UpperCamelCase				:      Any									=       WhisperProcessor(tokenizer=_snake_case			,  feature_extractor=_snake_case      )

							self.assertListEqual(
							    processor.model_input_names			,  feature_extractor.model_input_names			,  msg="`processor` and `feature_extractor` model input names do not match"			,  )
				def 	a_	(self      )     ->	Dict:
							__UpperCamelCase				:      Tuple									=       self.get_feature_extractor()
							__UpperCamelCase				:      Tuple									=       self.get_tokenizer()

							__UpperCamelCase				:      Optional[int]									=       WhisperProcessor(tokenizer=_snake_case			,  feature_extractor=_snake_case      )
							__UpperCamelCase				:      Any									=       processor.get_decoder_prompt_ids(task="transcribe"			,  no_timestamps=_snake_case      )

							self.assertIsInstance(_snake_case			,  _snake_case      )
							for ids in forced_decoder_ids:
										self.assertIsInstance(_snake_case			,  (list, tuple)      )

							__UpperCamelCase				:      List[str]									=       [TRANSCRIBE, NOTIMESTAMPS]
							self.assertListEqual([ids[-1] for ids in forced_decoder_ids]			,  _snake_case      )
				def 	a_	(self      )     ->	int:
							__UpperCamelCase				:      List[Any]									=       WhisperProcessor(tokenizer=self.get_tokenizer()			,  feature_extractor=self.get_feature_extractor()      )
							__UpperCamelCase				:      Optional[Any]									=       processor.get_prompt_ids("Mr. Quilter"      )
							__UpperCamelCase				:      List[Any]									=       processor.tokenizer.decode(_snake_case      )

							self.assertListEqual(prompt_ids.tolist()			,  [5_0_3_6_0, 1_7_7_0, 1_3, 2_2_6_4, 3_4_6, 3_5_3]      )
							self.assertEqual(_snake_case			,  "<|startofprev|> Mr. Quilter"      )
				def 	a_	(self      )     ->	Tuple:
							__UpperCamelCase				:      int									=       WhisperProcessor(tokenizer=self.get_tokenizer()			,  feature_extractor=self.get_feature_extractor()      )
							__UpperCamelCase				:      List[str]									=       processor.get_prompt_ids(""      )
							__UpperCamelCase				:      List[Any]									=       processor.tokenizer.decode(_snake_case      )

							self.assertListEqual(prompt_ids.tolist()			,  [5_0_3_6_0, 2_2_0]      )
							self.assertEqual(_snake_case			,  "<|startofprev|> "      )





				def 	a_	(self      )     ->	str:
							__UpperCamelCase				:      Any									=       WhisperProcessor(tokenizer=self.get_tokenizer()			,  feature_extractor=self.get_feature_extractor()      )

							def _test_prompt_error_raised_helper(_UpperCAmelCase			,  _UpperCAmelCase      ):
										with pytest.raises(_snake_case      ) as excinfo:
													processor.get_prompt_ids(_snake_case      )
										__UpperCamelCase				:      Dict									=       f"Encountered text in the prompt corresponding to disallowed special token: {special_token}."
										self.assertEqual(_snake_case			,  str(excinfo.value      )      )

							_test_prompt_error_raised_helper("<|startofprev|> test"			,  "<|startofprev|>"      )
							_test_prompt_error_raised_helper("test <|notimestamps|>"			,  "<|notimestamps|>"      )
							_test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>"			,  "<|zh|>"      )









'''simple docstring'''






from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


_lowerCAmelCase				     = logging.get_logger(__name__)

_lowerCAmelCase				     = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCAmelCase				     = {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
    '''tokenizer_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/tokenizer.json'''},
}

_lowerCAmelCase				     = {
    '''openai-gpt''': 512,
}



class  A		(			SCREAMING_SNAKE_CASE__		):




				'''simple docstring'''





				A				     =    VOCAB_FILES_NAMES
				A				     =    PRETRAINED_VOCAB_FILES_MAP
				A				     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A				     =    ["input_ids", "attention_mask"]
				A				     =    OpenAIGPTTokenizer
				def __init__(self			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase="<unk>"			,  **_UpperCAmelCase      )     ->	str:
							super().__init__(_UpperCAmelCase			,  _UpperCAmelCase			,  tokenizer_file=_UpperCAmelCase			,  unk_token=_UpperCAmelCase			,  **_UpperCAmelCase      )
				@property
				def 	a_	(self      )     ->	Dict:
							return True





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	Tuple[str]:
							__UpperCamelCase				:      Optional[Any]									=       self._tokenizer.model.save(_UpperCAmelCase			,  name=_UpperCAmelCase      )
							return tuple(_UpperCAmelCase      )



