




import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
    import torch


lowerCamelCase_  						=       random.Random()



def      UpperCAmelCase_   (     __UpperCamelCase,    __UpperCamelCase=1.0,    __UpperCamelCase=None,    __UpperCamelCase=None     ):
    if rng is None:
        SCREAMING_SNAKE_CASE__       =global_rng

    SCREAMING_SNAKE_CASE__       =[]
    for batch_idx in range(shape[0]     ):
        values.append([]     )
        for _ in range(shape[1]     ):
            values[-1].append(rng.random() * scale     )

    return values
@require_torch
class 				__a (			unittest.TestCase					):


     """simple docstring"""


     def __init__(     self      :			Any    ,_UpperCamelCase      :			Optional[int]    ,_UpperCamelCase      :			Union[str, Any]=7    ,_UpperCamelCase      :			Any=4_0_0    ,_UpperCamelCase      :			Optional[int]=2_0_0_0    ,_UpperCamelCase      :			str=1    ,_UpperCamelCase      :			Optional[Any]=0.0    ,_UpperCamelCase      :			Union[str, Any]=2_4_0_0_0    ,_UpperCamelCase      :			str=True    ,)		->       Tuple:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =parent
         SCREAMING_SNAKE_CASE__       =batch_size
         SCREAMING_SNAKE_CASE__       =min_seq_length
         SCREAMING_SNAKE_CASE__       =max_seq_length
         SCREAMING_SNAKE_CASE__       =(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
         SCREAMING_SNAKE_CASE__       =feature_size
         SCREAMING_SNAKE_CASE__       =padding_value
         SCREAMING_SNAKE_CASE__       =sampling_rate
         SCREAMING_SNAKE_CASE__       =return_attention_mask




     def   __A    (     self      :			int						)		->       Dict:


         '''simple docstring'''





         return {
             "feature_size": self.feature_size,
             "padding_value": self.padding_value,
             "sampling_rate": self.sampling_rate,
             "return_attention_mask": self.return_attention_mask,
         }




     def   __A    (     self      :			Union[str, Any]    ,_UpperCamelCase      :			int=False    ,_UpperCamelCase      :			List[Any]=False						)		->       int:


         '''simple docstring'''





         def _flatten(_UpperCamelCase      :			int						):
             return list(itertools.chain(*_UpperCamelCase						)						)

         if equal_length:
             SCREAMING_SNAKE_CASE__       =floats_list((self.batch_size, self.max_seq_length)						)
         else:
             # make sure that inputs increase in size
             SCREAMING_SNAKE_CASE__       =[
                 _flatten(floats_list((x, self.feature_size)						)						)
                 for x in range(self.min_seq_length    ,self.max_seq_length    ,self.seq_length_diff						)
             ]

         if numpify:
             SCREAMING_SNAKE_CASE__       =[np.asarray(_UpperCamelCase						) for x in audio_inputs]

         return audio_inputs






@require_torch
class 				__a (			__lowerCamelCase							,					unittest.TestCase					):


     """simple docstring"""


     _A    :   Optional[Any]     =  EncodecFeatureExtractor




     def   __A    (     self      :			Optional[Any]						)		->       List[str]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =EnCodecFeatureExtractionTester(self						)




     def   __A    (     self      :			Dict						)		->       List[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
         # create three inputs of length 800, 1000, and 1200
         SCREAMING_SNAKE_CASE__       =[floats_list((1, x)						)[0] for x in range(8_0_0    ,1_4_0_0    ,2_0_0						)]
         SCREAMING_SNAKE_CASE__       =[np.asarray(_UpperCamelCase						) for audio_input in audio_inputs]

         # Test not batched input
         SCREAMING_SNAKE_CASE__       =feat_extract(audio_inputs[0]    ,return_tensors="""np"""						).input_values
         SCREAMING_SNAKE_CASE__       =feat_extract(np_audio_inputs[0]    ,return_tensors="""np"""						).input_values
         self.assertTrue(np.allclose(_UpperCamelCase    ,_UpperCamelCase    ,atol=1e-3						)						)

         # Test batched
         SCREAMING_SNAKE_CASE__       =feat_extract(_UpperCamelCase    ,padding=_UpperCamelCase    ,return_tensors="""np"""						).input_values
         SCREAMING_SNAKE_CASE__       =feat_extract(_UpperCamelCase    ,padding=_UpperCamelCase    ,return_tensors="""np"""						).input_values
         for enc_seq_a, enc_seq_a in zip(_UpperCamelCase    ,_UpperCamelCase						):
             self.assertTrue(np.allclose(_UpperCamelCase    ,_UpperCamelCase    ,atol=1e-3						)						)




     def   __A    (     self      :			Any						)		->       Optional[int]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
         SCREAMING_SNAKE_CASE__       =np.random.rand(1_0_0						).astype(np.floataa						)
         SCREAMING_SNAKE_CASE__       =np_audio_inputs.tolist()

         for inputs in [py_audio_inputs, np_audio_inputs]:
             SCREAMING_SNAKE_CASE__       =feature_extractor.pad([{"""input_values""": inputs}]    ,return_tensors="""np"""						)
             self.assertTrue(np_processed.input_values.dtype == np.floataa						)
             SCREAMING_SNAKE_CASE__       =feature_extractor.pad([{"""input_values""": inputs}]    ,return_tensors="""pt"""						)
             self.assertTrue(pt_processed.input_values.dtype == torch.floataa						)




     def   __A    (     self      :			Tuple    ,_UpperCamelCase      :			int						)		->       List[str]:


         '''simple docstring'''





         from datasets import load_dataset

         SCREAMING_SNAKE_CASE__       =load_dataset("""hf-internal-testing/librispeech_asr_dummy"""    ,"""clean"""    ,split="""validation"""						)
         # automatic decoding with librispeech
         SCREAMING_SNAKE_CASE__       =ds.sort("""id"""						).select(range(_UpperCamelCase						)						)[:num_samples]["""audio"""]

         return [x["array"] for x in audio_samples]




     def   __A    (     self      :			List[str]						)		->       str:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =torch.tensor(
             [2.38_04e-03, 2.07_52e-03, 1.98_36e-03, 2.10_57e-03, 1.61_74e-03,
              3.05_18e-04, 9.15_53e-05, 3.35_69e-04, 9.76_56e-04, 1.83_11e-03,
              2.01_42e-03, 2.10_57e-03, 1.73_95e-03, 4.57_76e-04, -3.96_73e-04,
              4.57_76e-04, 1.00_71e-03, 9.15_53e-05, 4.88_28e-04, 1.15_97e-03,
              7.32_42e-04, 9.46_04e-04, 1.80_05e-03, 1.83_11e-03, 8.85_01e-04,
              4.27_25e-04, 4.88_28e-04, 7.32_42e-04, 1.09_86e-03, 2.10_57e-03]						)
         # fmt: on
         SCREAMING_SNAKE_CASE__       =self._load_datasamples(1						)
         SCREAMING_SNAKE_CASE__       =EncodecFeatureExtractor()
         SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(input_values.shape    ,(1, 1, 9_3_6_8_0)						)
         self.assertTrue(torch.allclose(input_values[0, 0, :3_0]    ,_UpperCamelCase    ,atol=1e-6						)						)




     def   __A    (     self      :			Optional[int]						)		->       List[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =torch.tensor(
             [2.38_04e-03, 2.07_52e-03, 1.98_36e-03, 2.10_57e-03, 1.61_74e-03,
              3.05_18e-04, 9.15_53e-05, 3.35_69e-04, 9.76_56e-04, 1.83_11e-03,
              2.01_42e-03, 2.10_57e-03, 1.73_95e-03, 4.57_76e-04, -3.96_73e-04,
              4.57_76e-04, 1.00_71e-03, 9.15_53e-05, 4.88_28e-04, 1.15_97e-03,
              7.32_42e-04, 9.46_04e-04, 1.80_05e-03, 1.83_11e-03, 8.85_01e-04,
              4.27_25e-04, 4.88_28e-04, 7.32_42e-04, 1.09_86e-03, 2.10_57e-03]						)
         # fmt: on
         SCREAMING_SNAKE_CASE__       =self._load_datasamples(1						)
         SCREAMING_SNAKE_CASE__       =[np.tile(input_audio[0][None]    ,reps=(2, 1)						)]
         input_audio[0][1] *= 0.5
         SCREAMING_SNAKE_CASE__       =EncodecFeatureExtractor(feature_size=2						)
         SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(input_values.shape    ,(1, 2, 9_3_6_8_0)						)
         self.assertTrue(torch.allclose(input_values[0, 0, :3_0]    ,_UpperCamelCase    ,atol=1e-6						)						)
         self.assertTrue(torch.allclose(input_values[0, 1, :3_0]    ,EXPECTED_INPUT_VALUES * 0.5    ,atol=1e-6						)						)




     def   __A    (     self      :			Optional[Any]						)		->       List[str]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =self._load_datasamples(2						)
         # would be easier if the stride was like
         SCREAMING_SNAKE_CASE__       =EncodecFeatureExtractor(feature_size=1    ,chunk_length_s=1    ,overlap=0.01						)

         # pad and trunc raise an error ?
         with self.assertRaisesRegex(
             _UpperCamelCase    ,"""^Both padding and truncation were set. Make sure you only set one.$"""    ,):
             SCREAMING_SNAKE_CASE__       =feature_extractor(
                 _UpperCamelCase    ,padding="""max_length"""    ,truncation=_UpperCamelCase    ,return_tensors="""pt"""						).input_values

         # truncate to chunk
         SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,truncation=_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(2, 1, 7_1_5_2_0)						)  # 2 chunks

         # force truncate to max_length
         SCREAMING_SNAKE_CASE__       =feature_extractor(
             _UpperCamelCase    ,truncation=_UpperCamelCase    ,max_length=4_8_0_0_0    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(2, 1, 4_8_0_0_0)						)

         # pad to chunk
         SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(padded_outputs.shape    ,(2, 1, 9_5_2_8_0)						)

         # pad to chunk
         SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(2, 1, 9_5_2_8_0)						)

         # force pad to max length
         SCREAMING_SNAKE_CASE__       =feature_extractor(
             _UpperCamelCase    ,padding="""max_length"""    ,max_length=1_0_0_0_0_0    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(2, 1, 1_0_0_0_0_0)						)

         # force no pad
         with self.assertRaisesRegex(
             _UpperCamelCase    ,"""^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"""    ,):
             SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values

         SCREAMING_SNAKE_CASE__       =feature_extractor(input_audio[0]    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(1, 1, 9_3_6_8_0)						)

         # no pad if no chunk_length_s
         SCREAMING_SNAKE_CASE__       =None
         with self.assertRaisesRegex(
             _UpperCamelCase    ,"""^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"""    ,):
             SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values

         SCREAMING_SNAKE_CASE__       =feature_extractor(input_audio[0]    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(1, 1, 9_3_6_8_0)						)

         # no pad if no overlap
         SCREAMING_SNAKE_CASE__       =2
         SCREAMING_SNAKE_CASE__       =None
         with self.assertRaisesRegex(
             _UpperCamelCase    ,"""^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"""    ,):
             SCREAMING_SNAKE_CASE__       =feature_extractor(_UpperCamelCase    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values

         SCREAMING_SNAKE_CASE__       =feature_extractor(input_audio[0]    ,padding=_UpperCamelCase    ,return_tensors="""pt"""						).input_values
         self.assertEquals(truncated_outputs.shape    ,(1, 1, 9_3_6_8_0)						)












import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
    from .tokenization_deberta_va import DebertaVaTokenizer
else:
    lowerCamelCase_  						=       None

lowerCamelCase_  						=       logging.get_logger(__name__)

lowerCamelCase_  						=       {"vocab_file": "spm.model", "tokenizer_file": "tokenizer.json"}

lowerCamelCase_  						=       {
    "vocab_file": {
        "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model"
        ),
        "microsoft/deberta-v2-xxlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model"
        ),
    }
}

lowerCamelCase_  						=       {
    "microsoft/deberta-v2-xlarge": 512,
    "microsoft/deberta-v2-xxlarge": 512,
    "microsoft/deberta-v2-xlarge-mnli": 512,
    "microsoft/deberta-v2-xxlarge-mnli": 512,
}

lowerCamelCase_  						=       {
    "microsoft/deberta-v2-xlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xlarge-mnli": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge-mnli": {"do_lower_case": False},
}
class 				__a (			__lowerCamelCase					):


     """simple docstring"""


     _A    :   str     =  VOCAB_FILES_NAMES
     _A    :   Union[str, Any]     =  PRETRAINED_VOCAB_FILES_MAP
     _A    :   Optional[int]     =  PRETRAINED_INIT_CONFIGURATION
     _A    :   Any     =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A    :   Optional[int]     =  DebertaVaTokenizer




     def __init__(     self      :			List[Any]    ,_UpperCamelCase      :			Tuple=None    ,_UpperCamelCase      :			str=None    ,_UpperCamelCase      :			Any=False    ,_UpperCamelCase      :			List[Any]=False    ,_UpperCamelCase      :			Any="[CLS]"    ,_UpperCamelCase      :			Optional[int]="[SEP]"    ,_UpperCamelCase      :			int="[UNK]"    ,_UpperCamelCase      :			int="[SEP]"    ,_UpperCamelCase      :			Tuple="[PAD]"    ,_UpperCamelCase      :			Optional[Any]="[CLS]"    ,_UpperCamelCase      :			Optional[int]="[MASK]"    ,**_UpperCamelCase      :			int    ,)		->       None:


         '''simple docstring'''





         super().__init__(
             _UpperCamelCase    ,tokenizer_file=_UpperCamelCase    ,do_lower_case=_UpperCamelCase    ,bos_token=_UpperCamelCase    ,eos_token=_UpperCamelCase    ,unk_token=_UpperCamelCase    ,sep_token=_UpperCamelCase    ,pad_token=_UpperCamelCase    ,cls_token=_UpperCamelCase    ,mask_token=_UpperCamelCase    ,split_by_punct=_UpperCamelCase    ,**_UpperCamelCase    ,)

         SCREAMING_SNAKE_CASE__       =do_lower_case
         SCREAMING_SNAKE_CASE__       =split_by_punct
         SCREAMING_SNAKE_CASE__       =vocab_file
         SCREAMING_SNAKE_CASE__       =False if not self.vocab_file else True




     def   __A    (     self      :			Optional[Any]    ,_UpperCamelCase      :			int    ,_UpperCamelCase      :			str=None						)		->       Dict:


         '''simple docstring'''





         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         SCREAMING_SNAKE_CASE__       =[self.cls_token_id]
         SCREAMING_SNAKE_CASE__       =[self.sep_token_id]
         return cls + token_ids_a + sep + token_ids_a + sep




     def   __A    (     self      :			List[str]    ,_UpperCamelCase      :			Any    ,_UpperCamelCase      :			List[Any]=None    ,_UpperCamelCase      :			Union[str, Any]=False						)		->       str:


         '''simple docstring'''





         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_UpperCamelCase    ,token_ids_a=_UpperCamelCase    ,already_has_special_tokens=_UpperCamelCase						)

         if token_ids_a is not None:
             return [1] + ([0] * len(_UpperCamelCase						)) + [1] + ([0] * len(_UpperCamelCase						)) + [1]
         return [1] + ([0] * len(_UpperCamelCase						)) + [1]




     def   __A    (     self      :			Union[str, Any]    ,_UpperCamelCase      :			List[str]    ,_UpperCamelCase      :			str=None						)		->       int:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =[self.sep_token_id]
         SCREAMING_SNAKE_CASE__       =[self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




     def   __A    (     self      :			int    ,_UpperCamelCase      :			str    ,_UpperCamelCase      :			Optional[str] = None						)		->       Tuple[str]:


         '''simple docstring'''





         if not self.can_save_slow_tokenizer:
             raise ValueError(
                 """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                 """tokenizer."""						)

         if not os.path.isdir(_UpperCamelCase						):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
             return
         SCREAMING_SNAKE_CASE__       =os.path.join(
             _UpperCamelCase    ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

         if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCamelCase						):
             copyfile(self.vocab_file    ,_UpperCamelCase						)

         return (out_vocab_file,)






