





"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

_lowerCamelCase  					=     logging.get_logger(__name__)

_lowerCamelCase  					=     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCamelCase  					=     {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

_lowerCamelCase  					=     {
    '''gpt2''': 10_24,
    '''gpt2-medium''': 10_24,
    '''gpt2-large''': 10_24,
    '''gpt2-xl''': 10_24,
    '''distilgpt2''': 10_24,
}




@lru_cache()
def 		lowerCAmelCase_     (  ):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	(
     list(range(ord('''!'''		)			,				ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)			,				ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)			,				ord('''ÿ'''		) + 1		)		)
 )
 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	bs[:]
 __SCREAMING_SNAKE_CASE			:      Tuple 							=	0
 for b in range(2**8		):
  if b not in bs:
   bs.append(lowercase_		)
   cs.append(2**8 + n		)
   n += 1
 __SCREAMING_SNAKE_CASE			:      Dict 							=	[chr(lowercase_		) for n in cs]
 return dict(zip(lowercase_			,				lowercase_		)		)




def 		lowerCAmelCase_     (  lowercase_       :							Optional[Any]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	set()
 __SCREAMING_SNAKE_CASE			:      Dict 							=	word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)		)
  __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	char
 return pairs






class       snake_case		(       __UpperCAmelCase       ):
      lowerCamelCase__      =			VOCAB_FILES_NAMES
      lowerCamelCase__      =			PRETRAINED_VOCAB_FILES_MAP
      lowerCamelCase__      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCamelCase__      =			['''input_ids''', '''attention_mask''']

      def __init__(      self       :Tuple						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Union[str, Any]						,							_lowerCamelCase       :Optional[int]="replace"						,							_lowerCamelCase       :Dict="<|endoftext|>"						,							_lowerCamelCase       :Any="<|endoftext|>"						,							_lowerCamelCase       :Optional[int]="<|endoftext|>"						,							_lowerCamelCase       :List[str]=None						,							_lowerCamelCase       :int=False						,							_lowerCamelCase       :Tuple=False						,							**_lowerCamelCase       :int						,							):
       __SCREAMING_SNAKE_CASE			:      Any 							=	AddedToken(_lowerCamelCase						,							lstrip=_lowerCamelCase						,							rstrip=_lowerCamelCase						) if isinstance(_lowerCamelCase						,							_lowerCamelCase						) else bos_token
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	AddedToken(_lowerCamelCase						,							lstrip=_lowerCamelCase						,							rstrip=_lowerCamelCase						) if isinstance(_lowerCamelCase						,							_lowerCamelCase						) else eos_token
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	AddedToken(_lowerCamelCase						,							lstrip=_lowerCamelCase						,							rstrip=_lowerCamelCase						) if isinstance(_lowerCamelCase						,							_lowerCamelCase						) else unk_token
       __SCREAMING_SNAKE_CASE			:      str 							=	AddedToken(_lowerCamelCase						,							lstrip=_lowerCamelCase						,							rstrip=_lowerCamelCase						) if isinstance(_lowerCamelCase						,							_lowerCamelCase						) else pad_token
       super().__init__(
           errors=_lowerCamelCase						,							unk_token=_lowerCamelCase						,							bos_token=_lowerCamelCase						,							eos_token=_lowerCamelCase						,							pad_token=_lowerCamelCase						,							add_prefix_space=_lowerCamelCase						,							add_bos_token=_lowerCamelCase						,							**_lowerCamelCase						,							)
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	add_bos_token

       with open(_lowerCamelCase						,							encoding='''utf-8'''						) as vocab_handle:
        __SCREAMING_SNAKE_CASE			:      str 							=	json.load(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      str 							=	{v: k for k, v in self.encoder.items()}
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	errors  # how to handle errors in decoding
       __SCREAMING_SNAKE_CASE			:      int 							=	bytes_to_unicode()
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	{v: k for k, v in self.byte_encoder.items()}
       with open(_lowerCamelCase						,							encoding='''utf-8'''						) as merges_handle:
        __SCREAMING_SNAKE_CASE			:      int 							=	merges_handle.read().split('''\n'''						)[1:-1]
       __SCREAMING_SNAKE_CASE			:      int 							=	[tuple(merge.split()						) for merge in bpe_merges]
       __SCREAMING_SNAKE_CASE			:      Any 							=	dict(zip(_lowerCamelCase						,							range(len(_lowerCamelCase						)						)						)						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	{}
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       __SCREAMING_SNAKE_CASE			:      Dict 							=	re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)

      @property
      def 						SCREAMING_SNAKE_CASE_      (      self       :int						):
       return len(self.encoder						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						):
       return dict(self.encoder						,							**self.added_tokens_encoder						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						,							_lowerCamelCase       :List[str]						):
       if token in self.cache:
        return self.cache[token]
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	tuple(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	get_pairs(_lowerCamelCase						)

       if not pairs:
        return token

       while True:
        __SCREAMING_SNAKE_CASE			:      str 							=	min(_lowerCamelCase						,							key=lambda _lowerCamelCase						: self.bpe_ranks.get(_lowerCamelCase						,							float('''inf'''						)						)						)
        if bigram not in self.bpe_ranks:
         break
        __SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE			:      List[str] 							=	bigram
        __SCREAMING_SNAKE_CASE			:      List[Any] 							=	[]
        __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	0
        while i < len(_lowerCamelCase						):
         try:
          __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	word.index(_lowerCamelCase						,							_lowerCamelCase						)
         except ValueError:
          new_word.extend(word[i:]						)
          break
         else:
          new_word.extend(word[i:j]						)
          __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	j

         if word[i] == first and i < len(_lowerCamelCase						) - 1 and word[i + 1] == second:
          new_word.append(first + second						)
          i += 2
         else:
          new_word.append(word[i]						)
          i += 1
        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	tuple(_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	new_word
        if len(_lowerCamelCase						) == 1:
         break
        else:
         __SCREAMING_SNAKE_CASE			:      int 							=	get_pairs(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	''' '''.join(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Any 							=	word
       return word

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						,							_lowerCamelCase       :str						,							_lowerCamelCase       :Dict=None						):
       if self.add_bos_token:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	[self.bos_token_id]
       else:
        __SCREAMING_SNAKE_CASE			:      Tuple 							=	[]

       __SCREAMING_SNAKE_CASE			:      Dict 							=	bos_token_ids + token_ids_a

       if token_ids_a is None:
        return output

       return output + bos_token_ids + token_ids_a

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :List[int]						,							_lowerCamelCase       :Optional[List[int]] = None						,							_lowerCamelCase       :bool = False						):
       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=_lowerCamelCase						,							token_ids_a=_lowerCamelCase						,							already_has_special_tokens=_lowerCamelCase						)

       if not self.add_bos_token:
        return super().get_special_tokens_mask(
            token_ids_a=_lowerCamelCase						,							token_ids_a=_lowerCamelCase						,							already_has_special_tokens=_lowerCamelCase						)

       if token_ids_a is None:
        return [1] + ([0] * len(_lowerCamelCase						))
       return [1] + ([0] * len(_lowerCamelCase						)) + [1] + ([0] * len(_lowerCamelCase						))

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						,							_lowerCamelCase       :Dict						):
       __SCREAMING_SNAKE_CASE			:      str 							=	[]
       for token in re.findall(self.pat						,							_lowerCamelCase						):
        __SCREAMING_SNAKE_CASE			:      str 							=	''''''.join(
            self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase						).split(''' '''						)						)
       return bpe_tokens

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :Optional[Any]						):
       return self.encoder.get(_lowerCamelCase						,							self.encoder.get(self.unk_token						)						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						,							_lowerCamelCase       :Union[str, Any]						):
       return self.decoder.get(_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	''''''.join(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8'''						,							errors=self.errors						)
       return text

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						,							_lowerCamelCase       :str						,							_lowerCamelCase       :Optional[str] = None						):
       if not os.path.isdir(_lowerCamelCase						):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
        return
       __SCREAMING_SNAKE_CASE			:      Dict 							=	os.path.join(
           _lowerCamelCase						,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	os.path.join(
           _lowerCamelCase						,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

       with open(_lowerCamelCase						,							'''w'''						,							encoding='''utf-8'''						) as f:
        f.write(json.dumps(self.encoder						,							indent=2						,							sort_keys=_lowerCamelCase						,							ensure_ascii=_lowerCamelCase						) + '''\n'''						)

       __SCREAMING_SNAKE_CASE			:      Any 							=	0
       with open(_lowerCamelCase						,							'''w'''						,							encoding='''utf-8'''						) as writer:
        writer.write('''#version: 0.2\n'''						)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,							key=lambda _lowerCamelCase						: kv[1]						):
         if index != token_index:
          logger.warning(
              f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              ''' Please check that the tokenizer is not corrupted!'''						)
          __SCREAMING_SNAKE_CASE			:      Tuple 							=	token_index
         writer.write(''' '''.join(_lowerCamelCase						) + '''\n'''						)
         index += 1

       return vocab_file, merge_file

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						,							_lowerCamelCase       :Union[str, Any]						,							_lowerCamelCase       :int=False						,							**_lowerCamelCase       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	kwargs.pop('''add_prefix_space'''						,							self.add_prefix_space						)
       if is_split_into_words or add_prefix_space:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	''' ''' + text
       return (text, kwargs)



      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						,							_lowerCamelCase       :"Conversation"						):
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	[]
       for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(_lowerCamelCase						,							add_special_tokens=_lowerCamelCase						) + [self.eos_token_id]						)
       if len(_lowerCamelCase						) > self.model_max_length:
        __SCREAMING_SNAKE_CASE			:      Dict 							=	input_ids[-self.model_max_length :]
       return input_ids








"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
    from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

    from .test_feature_extraction_speechta import floats_list


_lowerCamelCase  					=     get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')





@require_torch
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	tempfile.mkdtemp()

       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	SpeechTaTokenizer(_lowerCamelCase						)
       tokenizer.save_pretrained(self.tmpdirname						)

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	{
           '''feature_size''': 1,
           '''padding_value''': 0.0,
           '''sampling_rate''': 1_6_0_0_0,
           '''do_normalize''': False,
           '''num_mel_bins''': 8_0,
           '''hop_length''': 1_6,
           '''win_length''': 6_4,
           '''win_function''': '''hann_window''',
           '''fmin''': 8_0,
           '''fmax''': 7_6_0_0,
           '''mel_floor''': 1e-10,
           '''reduction_factor''': 2,
           '''return_attention_mask''': True,
       }

       __SCREAMING_SNAKE_CASE			:      Any 							=	os.path.join(self.tmpdirname						,							_lowerCamelCase						)
       with open(self.feature_extraction_file						,							'''w'''						,							encoding='''utf-8'''						) as fp:
        fp.write(json.dumps(_lowerCamelCase						) + '''\n'''						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						,							**_lowerCamelCase       :str						):
       return SpeechTaTokenizer.from_pretrained(self.tmpdirname						,							**_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						,							**_lowerCamelCase       :Dict						):
       return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname						,							**_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       shutil.rmtree(self.tmpdirname						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.get_tokenizer()
       __SCREAMING_SNAKE_CASE			:      int 							=	self.get_feature_extractor()

       __SCREAMING_SNAKE_CASE			:      Any 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       processor.save_pretrained(self.tmpdirname						)
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	SpeechTaProcessor.from_pretrained(self.tmpdirname						)

       self.assertEqual(processor.tokenizer.get_vocab()						,							tokenizer.get_vocab()						)
       self.assertIsInstance(processor.tokenizer						,							_lowerCamelCase						)

       self.assertEqual(processor.feature_extractor.to_json_string()						,							feature_extractor.to_json_string()						)
       self.assertIsInstance(processor.feature_extractor						,							_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	SpeechTaProcessor(tokenizer=self.get_tokenizer()						,							feature_extractor=self.get_feature_extractor()						)
       processor.save_pretrained(self.tmpdirname						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.get_tokenizer(bos_token='''(BOS)'''						,							eos_token='''(EOS)'''						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	self.get_feature_extractor(do_normalize=_lowerCamelCase						,							padding_value=1.0						)

       __SCREAMING_SNAKE_CASE			:      Any 							=	SpeechTaProcessor.from_pretrained(
           self.tmpdirname						,							bos_token='''(BOS)'''						,							eos_token='''(EOS)'''						,							do_normalize=_lowerCamelCase						,							padding_value=1.0						)

       self.assertEqual(processor.tokenizer.get_vocab()						,							tokenizer_add_kwargs.get_vocab()						)
       self.assertIsInstance(processor.tokenizer						,							_lowerCamelCase						)

       self.assertEqual(processor.feature_extractor.to_json_string()						,							feature_extractor_add_kwargs.to_json_string()						)
       self.assertIsInstance(processor.feature_extractor						,							_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      int 							=	floats_list((3, 1_0_0_0)						)

       __SCREAMING_SNAKE_CASE			:      int 							=	feature_extractor(audio=_lowerCamelCase						,							return_tensors='''np'''						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	processor(audio=_lowerCamelCase						,							return_tensors='''np'''						)

       for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()						,							input_processor[key].sum()						,							delta=1e-2						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Tuple						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      int 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Any 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	floats_list((3, 1_0_0_0)						)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	feature_extractor(audio_target=_lowerCamelCase						,							return_tensors='''np'''						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	processor(audio_target=_lowerCamelCase						,							return_tensors='''np'''						)

       for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()						,							input_processor[key].sum()						,							delta=1e-2						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      int 							=	'''This is a test string'''

       __SCREAMING_SNAKE_CASE			:      str 							=	processor(text=_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	tokenizer(_lowerCamelCase						)

       for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]						,							encoded_processor[key]						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      int 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      str 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Dict 							=	'''This is a test string'''

       __SCREAMING_SNAKE_CASE			:      str 							=	processor(text_target=_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Dict 							=	tokenizer(_lowerCamelCase						)

       for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]						,							encoded_processor[key]						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Dict 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	processor.batch_decode(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tokenizer.batch_decode(_lowerCamelCase						)

       self.assertListEqual(_lowerCamelCase						,							_lowerCamelCase						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      str 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      str 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       self.assertListEqual(
           processor.model_input_names						,							feature_extractor.model_input_names						,							msg='''`processor` and `feature_extractor` model input names do not match'''						,							)

