

'''simple docstring'''

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_      =							logging.get_logger(__name__)

A_      =							{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

A_      =							{
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
}

A_      =							{
    "roberta-base": 5_12,
    "roberta-large": 5_12,
    "roberta-large-mnli": 5_12,
    "distilroberta-base": 5_12,
    "roberta-base-openai-detector": 5_12,
    "roberta-large-openai-detector": 5_12,
}





@lru_cache()
def 				A_					(	):
  SCREAMING_SNAKE_CASE:int   =   (
      list(range(ord("!"	) ,    ord("~"	) + 1	)	) + list(range(ord("¡"	) ,    ord("¬"	) + 1	)	) + list(range(ord("®"	) ,    ord("ÿ"	) + 1	)	)
  )
  SCREAMING_SNAKE_CASE:List[Any]   =   bs[:]
  SCREAMING_SNAKE_CASE:Optional[int]   =   0
  for b in range(2**8	):
    if b not in bs:
      bs.append(snake_case	)
      cs.append(2**8 + n	)
      n += 1
  SCREAMING_SNAKE_CASE:Dict   =   [chr(snake_case	) for n in cs]
  return dict(zip(snake_case ,    snake_case	)	)





def 				A_					(	snake_case	):
  SCREAMING_SNAKE_CASE:Tuple   =   set()
  SCREAMING_SNAKE_CASE:str   =   word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)	)
    SCREAMING_SNAKE_CASE:Optional[int]   =   char
  return pairs





class 							_snake_case	(    _a  ):
   _A    :       List[str]      				=   VOCAB_FILES_NAMES
   _A    :       Optional[Any]      				=   PRETRAINED_VOCAB_FILES_MAP
   _A    :       Dict      				=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _A    :       Optional[int]      				=   ['''input_ids''', '''attention_mask''']
   def __init__(	self     : int       ,SCREAMING_SNAKE_CASE__     : Optional[Any]       ,SCREAMING_SNAKE_CASE__     : Union[str, Any]       ,SCREAMING_SNAKE_CASE__     : Union[str, Any]="replace"       ,SCREAMING_SNAKE_CASE__     : List[Any]="<s>"       ,SCREAMING_SNAKE_CASE__     : Dict="</s>"       ,SCREAMING_SNAKE_CASE__     : Any="</s>"       ,SCREAMING_SNAKE_CASE__     : List[str]="<s>"       ,SCREAMING_SNAKE_CASE__     : List[str]="<unk>"       ,SCREAMING_SNAKE_CASE__     : Union[str, Any]="<pad>"       ,SCREAMING_SNAKE_CASE__     : Dict="<mask>"       ,SCREAMING_SNAKE_CASE__     : Tuple=False       ,**SCREAMING_SNAKE_CASE__     : int       ,):
     SCREAMING_SNAKE_CASE:List[str]   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else bos_token
     SCREAMING_SNAKE_CASE:List[Any]   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else eos_token
     SCREAMING_SNAKE_CASE:Any   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else sep_token
     SCREAMING_SNAKE_CASE:Union[str, Any]   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else cls_token
     SCREAMING_SNAKE_CASE:List[Any]   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else unk_token
     SCREAMING_SNAKE_CASE:Union[str, Any]   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     SCREAMING_SNAKE_CASE:List[str]   =   AddedToken(SCREAMING_SNAKE_CASE__       ,lstrip=SCREAMING_SNAKE_CASE__       ,rstrip=SCREAMING_SNAKE_CASE__							) if isinstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							) else mask_token

     super().__init__(
         errors=SCREAMING_SNAKE_CASE__       ,bos_token=SCREAMING_SNAKE_CASE__       ,eos_token=SCREAMING_SNAKE_CASE__       ,unk_token=SCREAMING_SNAKE_CASE__       ,sep_token=SCREAMING_SNAKE_CASE__       ,cls_token=SCREAMING_SNAKE_CASE__       ,pad_token=SCREAMING_SNAKE_CASE__       ,mask_token=SCREAMING_SNAKE_CASE__       ,add_prefix_space=SCREAMING_SNAKE_CASE__       ,**SCREAMING_SNAKE_CASE__       ,)

     with open(SCREAMING_SNAKE_CASE__       ,encoding="utf-8"							) as vocab_handle:
       SCREAMING_SNAKE_CASE:Any   =   json.load(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]   =   {v: k for k, v in self.encoder.items()}
     SCREAMING_SNAKE_CASE:Tuple   =   errors  # how to handle errors in decoding
     SCREAMING_SNAKE_CASE:Any   =   bytes_to_unicode()
     SCREAMING_SNAKE_CASE:str   =   {v: k for k, v in self.byte_encoder.items()}
     with open(SCREAMING_SNAKE_CASE__       ,encoding="utf-8"							) as merges_handle:
       SCREAMING_SNAKE_CASE:Optional[int]   =   merges_handle.read().split("\n"							)[1:-1]
     SCREAMING_SNAKE_CASE:Tuple   =   [tuple(merge.split()							) for merge in bpe_merges]
     SCREAMING_SNAKE_CASE:int   =   dict(zip(SCREAMING_SNAKE_CASE__       ,range(len(SCREAMING_SNAKE_CASE__							)							)							)							)
     SCREAMING_SNAKE_CASE:int   =   {}
     SCREAMING_SNAKE_CASE:str   =   add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     SCREAMING_SNAKE_CASE:Optional[Any]   =   re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"							)
   @property
   def       __UpperCamelCase  (	self     : Tuple							):
     return len(self.encoder							)
   def       __UpperCamelCase  (	self     : Union[str, Any]							):
     return dict(self.encoder       ,**self.added_tokens_encoder							)
   def       __UpperCamelCase  (	self     : Union[str, Any]       ,SCREAMING_SNAKE_CASE__     : List[str]							):
     if token in self.cache:
       return self.cache[token]
     SCREAMING_SNAKE_CASE:Dict   =   tuple(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]   =   get_pairs(SCREAMING_SNAKE_CASE__							)

     if not pairs:
       return token

     while True:
       SCREAMING_SNAKE_CASE:List[Any]   =   min(SCREAMING_SNAKE_CASE__       ,key=lambda SCREAMING_SNAKE_CASE__							: self.bpe_ranks.get(SCREAMING_SNAKE_CASE__       ,float("inf"							)							)							)
       if bigram not in self.bpe_ranks:
         break
       SCREAMING_SNAKE_CASE ,      SCREAMING_SNAKE_CASE:Any   =   bigram
       SCREAMING_SNAKE_CASE:Optional[Any]   =   []
       SCREAMING_SNAKE_CASE:Tuple   =   0
       while i < len(SCREAMING_SNAKE_CASE__							):
         try:
           SCREAMING_SNAKE_CASE:Optional[int]   =   word.index(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							)
         except ValueError:
           new_word.extend(word[i:]							)
           break
         else:
           new_word.extend(word[i:j]							)
           SCREAMING_SNAKE_CASE:Dict   =   j

         if word[i] == first and i < len(SCREAMING_SNAKE_CASE__							) - 1 and word[i + 1] == second:
           new_word.append(first + second							)
           i += 2
         else:
           new_word.append(word[i]							)
           i += 1
       SCREAMING_SNAKE_CASE:int   =   tuple(SCREAMING_SNAKE_CASE__							)
       SCREAMING_SNAKE_CASE:int   =   new_word
       if len(SCREAMING_SNAKE_CASE__							) == 1:
         break
       else:
         SCREAMING_SNAKE_CASE:Dict   =   get_pairs(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]   =   " ".join(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Any   =   word
     return word
   def       __UpperCamelCase  (	self     : Optional[Any]       ,SCREAMING_SNAKE_CASE__     : Union[str, Any]							):
     SCREAMING_SNAKE_CASE:int   =   []
     for token in re.findall(self.pat       ,SCREAMING_SNAKE_CASE__							):
       SCREAMING_SNAKE_CASE:List[Any]   =   "".join(
           self.byte_encoder[b] for b in token.encode("utf-8"							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE__							).split(" "							)							)
     return bpe_tokens
   def       __UpperCamelCase  (	self     : List[str]       ,SCREAMING_SNAKE_CASE__     : List[str]							):
     return self.encoder.get(SCREAMING_SNAKE_CASE__       ,self.encoder.get(self.unk_token							)							)
   def       __UpperCamelCase  (	self     : Tuple       ,SCREAMING_SNAKE_CASE__     : Optional[int]							):
     return self.decoder.get(SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : int       ,SCREAMING_SNAKE_CASE__     : int							):
     SCREAMING_SNAKE_CASE:str   =   "".join(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]   =   bytearray([self.byte_decoder[c] for c in text]							).decode("utf-8"       ,errors=self.errors							)
     return text
   def       __UpperCamelCase  (	self     : Union[str, Any]       ,SCREAMING_SNAKE_CASE__     : str       ,SCREAMING_SNAKE_CASE__     : Optional[str] = None							):
     if not os.path.isdir(SCREAMING_SNAKE_CASE__							):
       logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
       return
     SCREAMING_SNAKE_CASE:Any   =   os.path.join(
         SCREAMING_SNAKE_CASE__       ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)
     SCREAMING_SNAKE_CASE:Union[str, Any]   =   os.path.join(
         SCREAMING_SNAKE_CASE__       ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]							)

     with open(SCREAMING_SNAKE_CASE__       ,"w"       ,encoding="utf-8"							) as f:
       f.write(json.dumps(self.encoder       ,indent=2       ,sort_keys=SCREAMING_SNAKE_CASE__       ,ensure_ascii=SCREAMING_SNAKE_CASE__							) + "\n"							)

     SCREAMING_SNAKE_CASE:Tuple   =   0
     with open(SCREAMING_SNAKE_CASE__       ,"w"       ,encoding="utf-8"							) as writer:
       writer.write("#version: 0.2\n"							)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,key=lambda SCREAMING_SNAKE_CASE__							: kv[1]							):
         if index != token_index:
           logger.warning(
               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
               " Please check that the tokenizer is not corrupted!"							)
           SCREAMING_SNAKE_CASE:Optional[Any]   =   token_index
         writer.write(" ".join(SCREAMING_SNAKE_CASE__							) + "\n"							)
         index += 1

     return vocab_file, merge_file
   def       __UpperCamelCase  (	self     : Dict       ,SCREAMING_SNAKE_CASE__     : List[int]       ,SCREAMING_SNAKE_CASE__     : Optional[List[int]] = None							):
     if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     SCREAMING_SNAKE_CASE:str   =   [self.cls_token_id]
     SCREAMING_SNAKE_CASE:Tuple   =   [self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep
   def       __UpperCamelCase  (	self     : Optional[Any]       ,SCREAMING_SNAKE_CASE__     : List[int]       ,SCREAMING_SNAKE_CASE__     : Optional[List[int]] = None       ,SCREAMING_SNAKE_CASE__     : bool = False							):
     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=SCREAMING_SNAKE_CASE__       ,token_ids_a=SCREAMING_SNAKE_CASE__       ,already_has_special_tokens=SCREAMING_SNAKE_CASE__							)

     if token_ids_a is None:
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE__							)) + [1]
     return [1] + ([0] * len(SCREAMING_SNAKE_CASE__							)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__							)) + [1]
   def       __UpperCamelCase  (	self     : List[Any]       ,SCREAMING_SNAKE_CASE__     : List[int]       ,SCREAMING_SNAKE_CASE__     : Optional[List[int]] = None							):
     SCREAMING_SNAKE_CASE:List[str]   =   [self.sep_token_id]
     SCREAMING_SNAKE_CASE:str   =   [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep							) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]







   def       __UpperCamelCase  (	self     : Tuple       ,SCREAMING_SNAKE_CASE__     : Dict       ,SCREAMING_SNAKE_CASE__     : Any=False       ,**SCREAMING_SNAKE_CASE__     : Union[str, Any]							):
     SCREAMING_SNAKE_CASE:List[Any]   =   kwargs.pop("add_prefix_space"       ,self.add_prefix_space							)
     if (is_split_into_words or add_prefix_space) and (len(SCREAMING_SNAKE_CASE__							) > 0 and not text[0].isspace()):
       SCREAMING_SNAKE_CASE:Dict   =   " " + text
     return (text, kwargs)









'''simple docstring'''

import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
    from transformers import WhisperFeatureExtractor, WhisperProcessor


A_      =							5_03_58
A_      =							5_03_62





@require_torch
@require_torchaudio
@require_sentencepiece
class 							_snake_case	(    unittest.TestCase  ):
   def       __UpperCamelCase  (	self     : Any							):
     SCREAMING_SNAKE_CASE:int   =   "openai/whisper-small.en"
     SCREAMING_SNAKE_CASE:Optional[Any]   =   tempfile.mkdtemp()
   def       __UpperCamelCase  (	self     : Any       ,**SCREAMING_SNAKE_CASE__     : List[str]							):
     return WhisperTokenizer.from_pretrained(self.checkpoint       ,**SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : List[str]       ,**SCREAMING_SNAKE_CASE__     : Dict							):
     return WhisperFeatureExtractor.from_pretrained(self.checkpoint       ,**SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : str							):
     shutil.rmtree(self.tmpdirname							)
   def       __UpperCamelCase  (	self     : Any							):
     SCREAMING_SNAKE_CASE:Dict   =   self.get_tokenizer()
     SCREAMING_SNAKE_CASE:Optional[int]   =   self.get_feature_extractor()

     SCREAMING_SNAKE_CASE:Dict   =   WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__       ,feature_extractor=SCREAMING_SNAKE_CASE__							)

     processor.save_pretrained(self.tmpdirname							)
     SCREAMING_SNAKE_CASE:List[Any]   =   WhisperProcessor.from_pretrained(self.tmpdirname							)

     self.assertEqual(processor.tokenizer.get_vocab()       ,tokenizer.get_vocab()							)
     self.assertIsInstance(processor.tokenizer       ,SCREAMING_SNAKE_CASE__							)

     self.assertEqual(processor.feature_extractor.to_json_string()       ,feature_extractor.to_json_string()							)
     self.assertIsInstance(processor.feature_extractor       ,SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : Union[str, Any]							):
     SCREAMING_SNAKE_CASE:int   =   WhisperProcessor(tokenizer=self.get_tokenizer()       ,feature_extractor=self.get_feature_extractor()							)
     processor.save_pretrained(self.tmpdirname							)

     SCREAMING_SNAKE_CASE:List[str]   =   self.get_tokenizer(bos_token="(BOS)"       ,eos_token="(EOS)"							)
     SCREAMING_SNAKE_CASE:Tuple   =   self.get_feature_extractor(do_normalize=SCREAMING_SNAKE_CASE__       ,padding_value=1.0							)

     SCREAMING_SNAKE_CASE:Any   =   WhisperProcessor.from_pretrained(
         self.tmpdirname       ,bos_token="(BOS)"       ,eos_token="(EOS)"       ,do_normalize=SCREAMING_SNAKE_CASE__       ,padding_value=1.0							)

     self.assertEqual(processor.tokenizer.get_vocab()       ,tokenizer_add_kwargs.get_vocab()							)
     self.assertIsInstance(processor.tokenizer       ,SCREAMING_SNAKE_CASE__							)

     self.assertEqual(processor.feature_extractor.to_json_string()       ,feature_extractor_add_kwargs.to_json_string()							)
     self.assertIsInstance(processor.feature_extractor       ,SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : List[Any]							):
     SCREAMING_SNAKE_CASE:Any   =   self.get_feature_extractor()
     SCREAMING_SNAKE_CASE:List[Any]   =   self.get_tokenizer()

     SCREAMING_SNAKE_CASE:Optional[Any]   =   WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__       ,feature_extractor=SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Any   =   floats_list((3, 1_000)							)

     SCREAMING_SNAKE_CASE:Any   =   feature_extractor(SCREAMING_SNAKE_CASE__       ,return_tensors="np"							)
     SCREAMING_SNAKE_CASE:List[str]   =   processor(SCREAMING_SNAKE_CASE__       ,return_tensors="np"							)

     for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum()       ,input_processor[key].sum()       ,delta=1e-2							)
   def       __UpperCamelCase  (	self     : List[Any]							):
     SCREAMING_SNAKE_CASE:int   =   self.get_feature_extractor()
     SCREAMING_SNAKE_CASE:Tuple   =   self.get_tokenizer()

     SCREAMING_SNAKE_CASE:Union[str, Any]   =   WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__       ,feature_extractor=SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:str   =   "This is a test string"

     SCREAMING_SNAKE_CASE:int   =   processor(text=SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:str   =   tokenizer(SCREAMING_SNAKE_CASE__							)

     for key in encoded_tok.keys():
       self.assertListEqual(encoded_tok[key]       ,encoded_processor[key]							)
   def       __UpperCamelCase  (	self     : str							):
     SCREAMING_SNAKE_CASE:List[Any]   =   self.get_feature_extractor()
     SCREAMING_SNAKE_CASE:List[str]   =   self.get_tokenizer()

     SCREAMING_SNAKE_CASE:Union[str, Any]   =   WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__       ,feature_extractor=SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Optional[Any]   =   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     SCREAMING_SNAKE_CASE:Any   =   processor.batch_decode(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Union[str, Any]   =   tokenizer.batch_decode(SCREAMING_SNAKE_CASE__							)

     self.assertListEqual(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : int							):
     SCREAMING_SNAKE_CASE:Tuple   =   self.get_feature_extractor()
     SCREAMING_SNAKE_CASE:Optional[int]   =   self.get_tokenizer()

     SCREAMING_SNAKE_CASE:Union[str, Any]   =   WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__       ,feature_extractor=SCREAMING_SNAKE_CASE__							)

     self.assertListEqual(
         processor.model_input_names       ,feature_extractor.model_input_names       ,msg="`processor` and `feature_extractor` model input names do not match"       ,)
   def       __UpperCamelCase  (	self     : int							):
     SCREAMING_SNAKE_CASE:Tuple   =   self.get_feature_extractor()
     SCREAMING_SNAKE_CASE:Any   =   self.get_tokenizer()

     SCREAMING_SNAKE_CASE:Any   =   WhisperProcessor(tokenizer=SCREAMING_SNAKE_CASE__       ,feature_extractor=SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]   =   processor.get_decoder_prompt_ids(task="transcribe"       ,no_timestamps=SCREAMING_SNAKE_CASE__							)

     self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__							)
     for ids in forced_decoder_ids:
       self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,(list, tuple)							)

     SCREAMING_SNAKE_CASE:Any   =   [TRANSCRIBE, NOTIMESTAMPS]
     self.assertListEqual([ids[-1] for ids in forced_decoder_ids]       ,SCREAMING_SNAKE_CASE__							)
   def       __UpperCamelCase  (	self     : Dict							):
     SCREAMING_SNAKE_CASE:Tuple   =   WhisperProcessor(tokenizer=self.get_tokenizer()       ,feature_extractor=self.get_feature_extractor()							)
     SCREAMING_SNAKE_CASE:Dict   =   processor.get_prompt_ids("Mr. Quilter"							)
     SCREAMING_SNAKE_CASE:int   =   processor.tokenizer.decode(SCREAMING_SNAKE_CASE__							)

     self.assertListEqual(prompt_ids.tolist()       ,[50_360, 1_770, 13, 2_264, 346, 353]							)
     self.assertEqual(SCREAMING_SNAKE_CASE__       ,"<|startofprev|> Mr. Quilter"							)
   def       __UpperCamelCase  (	self     : Union[str, Any]							):
     SCREAMING_SNAKE_CASE:Dict   =   WhisperProcessor(tokenizer=self.get_tokenizer()       ,feature_extractor=self.get_feature_extractor()							)
     SCREAMING_SNAKE_CASE:List[Any]   =   processor.get_prompt_ids(""							)
     SCREAMING_SNAKE_CASE:Tuple   =   processor.tokenizer.decode(SCREAMING_SNAKE_CASE__							)

     self.assertListEqual(prompt_ids.tolist()       ,[50_360, 220]							)
     self.assertEqual(SCREAMING_SNAKE_CASE__       ,"<|startofprev|> "							)







   def       __UpperCamelCase  (	self     : Union[str, Any]							):
     SCREAMING_SNAKE_CASE:Tuple   =   WhisperProcessor(tokenizer=self.get_tokenizer()       ,feature_extractor=self.get_feature_extractor()							)

     def _test_prompt_error_raised_helper(SCREAMING_SNAKE_CASE__     : int       ,SCREAMING_SNAKE_CASE__     : List[str]							):
       with pytest.raises(SCREAMING_SNAKE_CASE__							) as excinfo:
         processor.get_prompt_ids(SCREAMING_SNAKE_CASE__							)
       SCREAMING_SNAKE_CASE:List[Any]   =   F'''Encountered text in the prompt corresponding to disallowed special token: {special_token}.'''
       self.assertEqual(SCREAMING_SNAKE_CASE__       ,str(excinfo.value							)							)

     _test_prompt_error_raised_helper("<|startofprev|> test"       ,"<|startofprev|>"							)
     _test_prompt_error_raised_helper("test <|notimestamps|>"       ,"<|notimestamps|>"							)
     _test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>"       ,"<|zh|>"							)






