




import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
		from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


__a  :		int		     =   get_tests_dir('''fixtures/test_sentencepiece.model''')







@require_torch
@require_torchaudio
@require_sentencepiece
class 				UpperCAmelCase(					unittest.TestCase					):



	"""simple docstring"""






	def    __a     (					self    ) ->      List[str]:





								"""simple docstring"""


								lowercase__  :	int        =							tempfile.mkdtemp()

								lowercase__  :	int        =							["<s>", "<pad>", "</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est"]
								lowercase__  :	Tuple        =							dict(zip(__lowerCamelCase	,				range(len(__lowerCamelCase    )    )    )    )
								lowercase__  :	int        =							Path(self.tmpdirname    )
								save_json(__lowerCamelCase	,				save_dir / VOCAB_FILES_NAMES["vocab_file"]    )
								if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
															copyfile(__lowerCamelCase	,				save_dir / VOCAB_FILES_NAMES["spm_file"]    )

								lowercase__  :	str        =							SpeechaTextTokenizer.from_pretrained(self.tmpdirname    )
								tokenizer.save_pretrained(self.tmpdirname    )

								lowercase__  :	Optional[int]        =							{
								    "feature_size": 24,
								    "num_mel_bins": 24,
								    "padding_value": 0.0,
								    "sampling_rate": 16000,
								    "return_attention_mask": False,
								    "do_normalize": True,
								}
								save_json(__lowerCamelCase	,				save_dir / FEATURE_EXTRACTOR_NAME    )

	def    __a     (					self	,				**lowerCamelCase    ) ->      str:





								"""simple docstring"""


								return SpeechaTextTokenizer.from_pretrained(self.tmpdirname	,				**__lowerCamelCase    )

	def    __a     (					self	,				**lowerCamelCase    ) ->      Tuple:





								"""simple docstring"""


								return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname	,				**__lowerCamelCase    )

	def    __a     (					self    ) ->      int:





								"""simple docstring"""


								shutil.rmtree(self.tmpdirname    )

	def    __a     (					self    ) ->      Dict:





								"""simple docstring"""


								lowercase__  :	int        =							self.get_tokenizer()
								lowercase__  :	Union[str, Any]        =							self.get_feature_extractor()

								lowercase__  :	Union[str, Any]        =							SpeechaTextProcessor(tokenizer=__lowerCamelCase	,				feature_extractor=__lowerCamelCase    )

								processor.save_pretrained(self.tmpdirname    )
								lowercase__  :	Tuple        =							SpeechaTextProcessor.from_pretrained(self.tmpdirname    )

								self.assertEqual(processor.tokenizer.get_vocab()	,				tokenizer.get_vocab()    )
								self.assertIsInstance(processor.tokenizer	,				__lowerCamelCase    )

								self.assertEqual(processor.feature_extractor.to_json_string()	,				feature_extractor.to_json_string()    )
								self.assertIsInstance(processor.feature_extractor	,				__lowerCamelCase    )

	def    __a     (					self    ) ->      str:





								"""simple docstring"""


								lowercase__  :	str        =							SpeechaTextProcessor(
								    tokenizer=self.get_tokenizer()	,				feature_extractor=self.get_feature_extractor()    )
								processor.save_pretrained(self.tmpdirname    )

								lowercase__  :	List[str]        =							self.get_tokenizer(bos_token="(BOS)"	,				eos_token="(EOS)"    )
								lowercase__  :	Dict        =							self.get_feature_extractor(do_normalize=__lowerCamelCase	,				padding_value=1.0    )

								lowercase__  :	List[str]        =							SpeechaTextProcessor.from_pretrained(
								    self.tmpdirname	,				bos_token="(BOS)"	,				eos_token="(EOS)"	,				do_normalize=__lowerCamelCase	,				padding_value=1.0    )

								self.assertEqual(processor.tokenizer.get_vocab()	,				tokenizer_add_kwargs.get_vocab()    )
								self.assertIsInstance(processor.tokenizer	,				__lowerCamelCase    )

								self.assertEqual(processor.feature_extractor.to_json_string()	,				feature_extractor_add_kwargs.to_json_string()    )
								self.assertIsInstance(processor.feature_extractor	,				__lowerCamelCase    )

	def    __a     (					self    ) ->      Union[str, Any]:





								"""simple docstring"""


								lowercase__  :	Union[str, Any]        =							self.get_feature_extractor()
								lowercase__  :	List[Any]        =							self.get_tokenizer()

								lowercase__  :	List[Any]        =							SpeechaTextProcessor(tokenizer=__lowerCamelCase	,				feature_extractor=__lowerCamelCase    )

								lowercase__  :	Optional[Any]        =							floats_list((3, 1000)    )

								lowercase__  :	Tuple        =							feature_extractor(__lowerCamelCase	,				return_tensors="np"    )
								lowercase__  :	Optional[int]        =							processor(__lowerCamelCase	,				return_tensors="np"    )

								for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()	,				input_processor[key].sum()	,				delta=1E-2    )

	def    __a     (					self    ) ->      Any:





								"""simple docstring"""


								lowercase__  :	int        =							self.get_feature_extractor()
								lowercase__  :	List[Any]        =							self.get_tokenizer()

								lowercase__  :	Optional[int]        =							SpeechaTextProcessor(tokenizer=__lowerCamelCase	,				feature_extractor=__lowerCamelCase    )

								lowercase__  :	int        =							"This is a test string"

								lowercase__  :	Tuple        =							processor(text=__lowerCamelCase    )

								lowercase__  :	Optional[int]        =							tokenizer(__lowerCamelCase    )

								for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]	,				encoded_processor[key]    )

	def    __a     (					self    ) ->      List[str]:





								"""simple docstring"""


								lowercase__  :	Union[str, Any]        =							self.get_feature_extractor()
								lowercase__  :	int        =							self.get_tokenizer()

								lowercase__  :	str        =							SpeechaTextProcessor(tokenizer=__lowerCamelCase	,				feature_extractor=__lowerCamelCase    )

								lowercase__  :	Union[str, Any]        =							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								lowercase__  :	str        =							processor.batch_decode(__lowerCamelCase    )
								lowercase__  :	Optional[int]        =							tokenizer.batch_decode(__lowerCamelCase    )

								self.assertListEqual(__lowerCamelCase	,				__lowerCamelCase    )







	def    __a     (					self    ) ->      Optional[Any]:





								"""simple docstring"""


								lowercase__  :	Any        =							self.get_feature_extractor()
								lowercase__  :	Optional[Any]        =							self.get_tokenizer()

								lowercase__  :	List[Any]        =							SpeechaTextProcessor(tokenizer=__lowerCamelCase	,				feature_extractor=__lowerCamelCase    )

								self.assertListEqual(
								    processor.model_input_names	,				feature_extractor.model_input_names	,				msg="`processor` and `feature_extractor` model input names do not match"	,				)





import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
						from .tokenization_deberta_va import DebertaVaTokenizer
else:
						__a  :		Optional[Any]		     =   None

__a  :		List[str]		     =   logging.get_logger(__name__)

__a  :		Dict		     =   {'''vocab_file''': '''spm.model''', '''tokenizer_file''': '''tokenizer.json'''}

__a  :		int		     =   {
    '''vocab_file''': {
        '''microsoft/deberta-v2-xlarge''': '''https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model''',
        '''microsoft/deberta-v2-xxlarge''': '''https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model''',
        '''microsoft/deberta-v2-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'''
        ),
        '''microsoft/deberta-v2-xxlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'''
        ),
    }
}

__a  :		Any		     =   {
    '''microsoft/deberta-v2-xlarge''': 5_1_2,
    '''microsoft/deberta-v2-xxlarge''': 5_1_2,
    '''microsoft/deberta-v2-xlarge-mnli''': 5_1_2,
    '''microsoft/deberta-v2-xxlarge-mnli''': 5_1_2,
}

__a  :		Union[str, Any]		     =   {
    '''microsoft/deberta-v2-xlarge''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xxlarge''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xlarge-mnli''': {'''do_lower_case''': False},
    '''microsoft/deberta-v2-xxlarge-mnli''': {'''do_lower_case''': False},
}







class 				UpperCAmelCase(					snake_case_					):



	"""simple docstring"""






	a     : List[Any]	  =					VOCAB_FILES_NAMES
	a     : Optional[int]	  =					PRETRAINED_VOCAB_FILES_MAP
	a     : int	  =					PRETRAINED_INIT_CONFIGURATION
	a     : List[str]	  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a     : int	  =					DebertaVaTokenizer

	def __init__(					self	,				lowerCamelCase=None	,				lowerCamelCase=None	,				lowerCamelCase=False	,				lowerCamelCase=False	,				lowerCamelCase="[CLS]"	,				lowerCamelCase="[SEP]"	,				lowerCamelCase="[UNK]"	,				lowerCamelCase="[SEP]"	,				lowerCamelCase="[PAD]"	,				lowerCamelCase="[CLS]"	,				lowerCamelCase="[MASK]"	,				**lowerCamelCase	,				) ->      None:





								"""simple docstring"""


								super().__init__(
								    lowerCamelCase	,				tokenizer_file=lowerCamelCase	,				do_lower_case=lowerCamelCase	,				bos_token=lowerCamelCase	,				eos_token=lowerCamelCase	,				unk_token=lowerCamelCase	,				sep_token=lowerCamelCase	,				pad_token=lowerCamelCase	,				cls_token=lowerCamelCase	,				mask_token=lowerCamelCase	,				split_by_punct=lowerCamelCase	,				**lowerCamelCase	,				)

								lowercase__  :	Optional[int]        =							do_lower_case
								lowercase__  :	Optional[Any]        =							split_by_punct
								lowercase__  :	Tuple        =							vocab_file
								lowercase__  :	Optional[int]        =							False if not self.vocab_file else True

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None    ) ->      List[Any]:





								"""simple docstring"""


								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowercase__  :	Union[str, Any]        =							[self.cls_token_id]
								lowercase__  :	Union[str, Any]        =							[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None	,				lowerCamelCase=False    ) ->      Union[str, Any]:





								"""simple docstring"""


								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCamelCase	,				token_ids_a=lowerCamelCase	,				already_has_special_tokens=lowerCamelCase    )

								if token_ids_a is not None:
															return [1] + ([0] * len(lowerCamelCase    )) + [1] + ([0] * len(lowerCamelCase    )) + [1]
								return [1] + ([0] * len(lowerCamelCase    )) + [1]

	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase=None    ) ->      Optional[int]:





								"""simple docstring"""


								lowercase__  :	Any        =							[self.sep_token_id]
								lowercase__  :	List[str]        =							[self.cls_token_id]
								if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







	def    __a     (					self	,				lowerCamelCase	,				lowerCamelCase = None    ) ->      Tuple[str]:





								"""simple docstring"""


								if not self.can_save_slow_tokenizer:
															raise ValueError(
															    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
															    "tokenizer."    )

								if not os.path.isdir(lowerCamelCase    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
															return
								lowercase__  :	int        =							os.path.join(
								    lowerCamelCase	,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

								if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase    ):
															copyfile(self.vocab_file	,				lowerCamelCase    )

								return (out_vocab_file,)