


import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
   from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

   from .test_feature_extraction_speechta import floats_list


_snake_case					=					get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')
@require_torch
class      UpperCAmelCase_ ( unittest.TestCase   ):

   '''simple docstring'''




   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: List[str]   =    tempfile.mkdtemp()

     lowerCamelCase			: List[str]   =    SpeechTaTokenizer(__A			)
     tokenizer.save_pretrained(self.tmpdirname			)

     lowerCamelCase			: Optional[int]   =    {
         "feature_size": 1,
         "padding_value": 0.0,
         "sampling_rate": 1_6000,
         "do_normalize": False,
         "num_mel_bins": 80,
         "hop_length": 16,
         "win_length": 64,
         "win_function": "hann_window",
         "fmin": 80,
         "fmax": 7600,
         "mel_floor": 1e-10,
         "reduction_factor": 2,
         "return_attention_mask": True,
     }

     lowerCamelCase			: Optional[Any]   =    os.path.join(self.tmpdirname						,    __A			)
     with open(self.feature_extraction_file						,    "w"						,    encoding="utf-8"			) as fp:
       fp.write(json.dumps(__A			) + "\n"			)






   def _snake_case   (		self						,    **__A			):






     """simple docstring"""





     return SpeechTaTokenizer.from_pretrained(self.tmpdirname						,    **__A			)






   def _snake_case   (		self						,    **__A			):






     """simple docstring"""





     return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname						,    **__A			)






   def _snake_case   (		self			):






     """simple docstring"""





     shutil.rmtree(self.tmpdirname			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: List[Any]   =    self.get_tokenizer()
     lowerCamelCase			: Tuple   =    self.get_feature_extractor()

     lowerCamelCase			: List[str]   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     processor.save_pretrained(self.tmpdirname			)
     lowerCamelCase			: int   =    SpeechTaProcessor.from_pretrained(self.tmpdirname			)

     self.assertEqual(processor.tokenizer.get_vocab()						,    tokenizer.get_vocab()			)
     self.assertIsInstance(processor.tokenizer						,    __A			)

     self.assertEqual(processor.feature_extractor.to_json_string()						,    feature_extractor.to_json_string()			)
     self.assertIsInstance(processor.feature_extractor						,    __A			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: List[str]   =    SpeechTaProcessor(tokenizer=self.get_tokenizer()						,    feature_extractor=self.get_feature_extractor()			)
     processor.save_pretrained(self.tmpdirname			)

     lowerCamelCase			: Optional[Any]   =    self.get_tokenizer(bos_token="(BOS)"						,    eos_token="(EOS)"			)
     lowerCamelCase			: str   =    self.get_feature_extractor(do_normalize=__A						,    padding_value=1.0			)

     lowerCamelCase			: str   =    SpeechTaProcessor.from_pretrained(
         self.tmpdirname						,    bos_token="(BOS)"						,    eos_token="(EOS)"						,    do_normalize=__A						,    padding_value=1.0			)

     self.assertEqual(processor.tokenizer.get_vocab()						,    tokenizer_add_kwargs.get_vocab()			)
     self.assertIsInstance(processor.tokenizer						,    __A			)

     self.assertEqual(processor.feature_extractor.to_json_string()						,    feature_extractor_add_kwargs.to_json_string()			)
     self.assertIsInstance(processor.feature_extractor						,    __A			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Union[str, Any]   =    self.get_feature_extractor()
     lowerCamelCase			: List[Any]   =    self.get_tokenizer()

     lowerCamelCase			: Optional[int]   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     lowerCamelCase			: List[str]   =    floats_list((3, 1000)			)

     lowerCamelCase			: Optional[Any]   =    feature_extractor(audio=__A						,    return_tensors="np"			)
     lowerCamelCase			: Optional[Any]   =    processor(audio=__A						,    return_tensors="np"			)

     for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum()						,    input_processor[key].sum()						,    delta=1e-2			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Optional[int]   =    self.get_feature_extractor()
     lowerCamelCase			: Optional[int]   =    self.get_tokenizer()

     lowerCamelCase			: Optional[Any]   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     lowerCamelCase			: Optional[Any]   =    floats_list((3, 1000)			)

     lowerCamelCase			: Optional[int]   =    feature_extractor(audio_target=__A						,    return_tensors="np"			)
     lowerCamelCase			: Dict   =    processor(audio_target=__A						,    return_tensors="np"			)

     for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum()						,    input_processor[key].sum()						,    delta=1e-2			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: str   =    self.get_feature_extractor()
     lowerCamelCase			: Optional[int]   =    self.get_tokenizer()

     lowerCamelCase			: Tuple   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     lowerCamelCase			: List[Any]   =    "This is a test string"

     lowerCamelCase			: str   =    processor(text=__A			)
     lowerCamelCase			: int   =    tokenizer(__A			)

     for key in encoded_tok.keys():
       self.assertListEqual(encoded_tok[key]						,    encoded_processor[key]			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Dict   =    self.get_feature_extractor()
     lowerCamelCase			: Optional[int]   =    self.get_tokenizer()

     lowerCamelCase			: Optional[Any]   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     lowerCamelCase			: Dict   =    "This is a test string"

     lowerCamelCase			: Tuple   =    processor(text_target=__A			)
     lowerCamelCase			: Union[str, Any]   =    tokenizer(__A			)

     for key in encoded_tok.keys():
       self.assertListEqual(encoded_tok[key]						,    encoded_processor[key]			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Any   =    self.get_feature_extractor()
     lowerCamelCase			: Tuple   =    self.get_tokenizer()

     lowerCamelCase			: int   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     lowerCamelCase			: Dict   =    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     lowerCamelCase			: Any   =    processor.batch_decode(__A			)
     lowerCamelCase			: int   =    tokenizer.batch_decode(__A			)

     self.assertListEqual(__A						,    __A			)






   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Optional[int]   =    self.get_feature_extractor()
     lowerCamelCase			: Any   =    self.get_tokenizer()

     lowerCamelCase			: Optional[Any]   =    SpeechTaProcessor(tokenizer=__A						,    feature_extractor=__A			)

     self.assertListEqual(
         processor.model_input_names						,    feature_extractor.model_input_names						,    msg="`processor` and `feature_extractor` model input names do not match"						,    )







from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy
class      UpperCAmelCase_ ( UpperCamelCase   ):

   '''simple docstring'''




   __A			:				Union[str, Any]							=		"MarkupLMFeatureExtractor"
   __A			:				List[str]							=		("MarkupLMTokenizer", "MarkupLMTokenizerFast")
   __A			:				str							=		True






   def __call__(		self						,    __A=None						,    __A=None						,    __A=None						,    __A=None						,    __A=None						,    __A = True						,    __A = False						,    __A = None						,    __A = None						,    __A = 0						,    __A = None						,    __A = None						,    __A = None						,    __A = False						,    __A = False						,    __A = False						,    __A = False						,    __A = True						,    __A = None						,    **__A						,    ):






     """simple docstring"""





     if self.parse_html:
       if html_strings is None:
         raise ValueError("Make sure to pass HTML strings in case `parse_html` is set to `True`"			)

       if nodes is not None or xpaths is not None or node_labels is not None:
         raise ValueError(
             "Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"			)

       lowerCamelCase			: Any   =    self.feature_extractor(__A			)
       lowerCamelCase			: Optional[int]   =    features["nodes"]
       lowerCamelCase			: Any   =    features["xpaths"]
     else:
       if html_strings is not None:
         raise ValueError("You have passed HTML strings but `parse_html` is set to `False`."			)
       if nodes is None or xpaths is None:
         raise ValueError("Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"			)

        # # second, apply the tokenizer
     if questions is not None and self.parse_html:
       if isinstance(__A						,    __A			):
         lowerCamelCase			: Tuple   =    [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

     lowerCamelCase			: Tuple   =    self.tokenizer(
         text=questions if questions is not None else nodes						,    text_pair=nodes if questions is not None else None						,    xpaths=__A						,    node_labels=__A						,    add_special_tokens=__A						,    padding=__A						,    truncation=__A						,    max_length=__A						,    stride=__A						,    pad_to_multiple_of=__A						,    return_token_type_ids=__A						,    return_attention_mask=__A						,    return_overflowing_tokens=__A						,    return_special_tokens_mask=__A						,    return_offsets_mapping=__A						,    return_length=__A						,    verbose=__A						,    return_tensors=__A						,    **__A						,    )

     return encoded_inputs






   def _snake_case   (		self						,    *__A						,    **__A			):






     """simple docstring"""





     return self.tokenizer.batch_decode(*__A						,    **__A			)






   def _snake_case   (		self						,    *__A						,    **__A			):






     """simple docstring"""





     return self.tokenizer.decode(*__A						,    **__A			)






   @property
   def _snake_case   (		self			):






     """simple docstring"""





     lowerCamelCase			: Dict   =    self.tokenizer.model_input_names
     return tokenizer_input_names



