




'''simple docstring'''







import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
   import torch


A			   =random.Random()





def 						snake_case_							(_a     :							Tuple   ,       _a     :							List[str]=1.0   ,       _a     :							Optional[Any]=None   ,       _a     :							Tuple=None					):
    if rng is None:
        UpperCAmelCase      	=		global_rng

    UpperCAmelCase      	=		[]
    for batch_idx in range(shape[0]					):
        values.append([]					)
        for _ in range(shape[1]					):
            values[-1].append(rng.random() * scale					)

    return values


@require_torch
class       _a  (     unittest.TestCase			):
       def __init__(     self						:     Dict       ,    lowercase						:     List[Any]       ,    lowercase						:     Union[str, Any]=7       ,    lowercase						:     List[Any]=400       ,    lowercase						:     int=2_000       ,    lowercase						:     List[Any]=1       ,    lowercase						:     str=0.0       ,    lowercase						:     Dict=24_000       ,    lowercase						:     Any=True       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		parent
           UpperCAmelCase      	=		batch_size
           UpperCAmelCase      	=		min_seq_length
           UpperCAmelCase      	=		max_seq_length
           UpperCAmelCase      	=		(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
           UpperCAmelCase      	=		feature_size
           UpperCAmelCase      	=		padding_value
           UpperCAmelCase      	=		sampling_rate
           UpperCAmelCase      	=		return_attention_mask
       def    A				(     self						:     Any    ):



           '''simple docstring'''

           return {
               "feature_size": self.feature_size,
               "padding_value": self.padding_value,
               "sampling_rate": self.sampling_rate,
               "return_attention_mask": self.return_attention_mask,
           }







       def    A				(     self						:     Dict       ,    lowercase						:     Optional[int]=False       ,    lowercase						:     List[Any]=False    ):



           '''simple docstring'''

           def _flatten(lowercase						:     List[str]    ):
               return list(itertools.chain(*lowercase    )    )

           if equal_length:
               UpperCAmelCase      	=		floats_list((self.batch_size, self.max_seq_length)    )
           else:
               # make sure that inputs increase in size
               UpperCAmelCase      	=		[
                   _flatten(floats_list((x, self.feature_size)    )    )
                   for x in range(self.min_seq_length       ,    self.max_seq_length       ,    self.seq_length_diff    )
               ]

           if numpify:
               UpperCAmelCase      	=		[np.asarray(lowercase    ) for x in audio_inputs]

           return audio_inputs






@require_torch
class       _a  (     __a							,							unittest.TestCase			):
       __a    :					List[Any]           = EncodecFeatureExtractor
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		EnCodecFeatureExtractionTester(self    )
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()    )
           # create three inputs of length 800, 1000, and 1200
           UpperCAmelCase      	=		[floats_list((1, x)    )[0] for x in range(800       ,    1_400       ,    200    )]
           UpperCAmelCase      	=		[np.asarray(lowercase    ) for audio_input in audio_inputs]

           # Test not batched input
           UpperCAmelCase      	=		feat_extract(audio_inputs[0]       ,    return_tensors='''np'''    ).input_values
           UpperCAmelCase      	=		feat_extract(np_audio_inputs[0]       ,    return_tensors='''np'''    ).input_values
           self.assertTrue(np.allclose(lowercase       ,    lowercase       ,    atol=1E-3    )    )

           # Test batched
           UpperCAmelCase      	=		feat_extract(lowercase       ,    padding=lowercase       ,    return_tensors='''np'''    ).input_values
           UpperCAmelCase      	=		feat_extract(lowercase       ,    padding=lowercase       ,    return_tensors='''np'''    ).input_values
           for enc_seq_a, enc_seq_a in zip(lowercase       ,    lowercase    ):
               self.assertTrue(np.allclose(lowercase       ,    lowercase       ,    atol=1E-3    )    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()    )
           UpperCAmelCase      	=		np.random.rand(100    ).astype(np.floataa    )
           UpperCAmelCase      	=		np_audio_inputs.tolist()

           for inputs in [py_audio_inputs, np_audio_inputs]:
               UpperCAmelCase      	=		feature_extractor.pad([{'''input_values''': inputs}]       ,    return_tensors='''np'''    )
               self.assertTrue(np_processed.input_values.dtype == np.floataa    )
               UpperCAmelCase      	=		feature_extractor.pad([{'''input_values''': inputs}]       ,    return_tensors='''pt'''    )
               self.assertTrue(pt_processed.input_values.dtype == torch.floataa    )
       def    A				(     self						:     Optional[Any]       ,    lowercase						:     str    ):



           '''simple docstring'''

           from datasets import load_dataset

           UpperCAmelCase      	=		load_dataset('''hf-internal-testing/librispeech_asr_dummy'''       ,    '''clean'''       ,    split='''validation'''    )
           # automatic decoding with librispeech
           UpperCAmelCase      	=		ds.sort('''id'''    ).select(range(lowercase    )    )[:num_samples]['''audio''']

           return [x["array"] for x in audio_samples]
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		torch.tensor(
               [2.3_804E-03, 2.0_752E-03, 1.9_836E-03, 2.1_057E-03, 1.6_174E-03,
                3.0_518E-04, 9.1_553E-05, 3.3_569E-04, 9.7_656E-04, 1.8_311E-03,
                2.0_142E-03, 2.1_057E-03, 1.7_395E-03, 4.5_776E-04, -3.9_673E-04,
                4.5_776E-04, 1.0_071E-03, 9.1_553E-05, 4.8_828E-04, 1.1_597E-03,
                7.3_242E-04, 9.4_604E-04, 1.8_005E-03, 1.8_311E-03, 8.8_501E-04,
                4.2_725E-04, 4.8_828E-04, 7.3_242E-04, 1.0_986E-03, 2.1_057E-03]    )
           # fmt: on
           UpperCAmelCase      	=		self._load_datasamples(1    )
           UpperCAmelCase      	=		EncodecFeatureExtractor()
           UpperCAmelCase      	=		feature_extractor(lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(input_values.shape       ,    (1, 1, 93_680)    )
           self.assertTrue(torch.allclose(input_values[0, 0, :30]       ,    lowercase       ,    atol=1E-6    )    )
       def    A				(     self						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase      	=		torch.tensor(
               [2.3_804E-03, 2.0_752E-03, 1.9_836E-03, 2.1_057E-03, 1.6_174E-03,
                3.0_518E-04, 9.1_553E-05, 3.3_569E-04, 9.7_656E-04, 1.8_311E-03,
                2.0_142E-03, 2.1_057E-03, 1.7_395E-03, 4.5_776E-04, -3.9_673E-04,
                4.5_776E-04, 1.0_071E-03, 9.1_553E-05, 4.8_828E-04, 1.1_597E-03,
                7.3_242E-04, 9.4_604E-04, 1.8_005E-03, 1.8_311E-03, 8.8_501E-04,
                4.2_725E-04, 4.8_828E-04, 7.3_242E-04, 1.0_986E-03, 2.1_057E-03]    )
           # fmt: on
           UpperCAmelCase      	=		self._load_datasamples(1    )
           UpperCAmelCase      	=		[np.tile(input_audio[0][None]       ,    reps=(2, 1)    )]
           input_audio[0][1] *= 0.5
           UpperCAmelCase      	=		EncodecFeatureExtractor(feature_size=2    )
           UpperCAmelCase      	=		feature_extractor(lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(input_values.shape       ,    (1, 2, 93_680)    )
           self.assertTrue(torch.allclose(input_values[0, 0, :30]       ,    lowercase       ,    atol=1E-6    )    )
           self.assertTrue(torch.allclose(input_values[0, 1, :30]       ,    EXPECTED_INPUT_VALUES * 0.5       ,    atol=1E-6    )    )







       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self._load_datasamples(2    )
           # would be easier if the stride was like
           UpperCAmelCase      	=		EncodecFeatureExtractor(feature_size=1       ,    chunk_length_s=1       ,    overlap=0.01    )

           # pad and trunc raise an error ?
           with self.assertRaisesRegex(
               lowercase       ,    '''^Both padding and truncation were set. Make sure you only set one.$'''       ,    ):
               UpperCAmelCase      	=		feature_extractor(
                   lowercase       ,    padding='''max_length'''       ,    truncation=lowercase       ,    return_tensors='''pt'''    ).input_values

           # truncate to chunk
           UpperCAmelCase      	=		feature_extractor(lowercase       ,    truncation=lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (2, 1, 71_520)    )  # 2 chunks

           # force truncate to max_length
           UpperCAmelCase      	=		feature_extractor(
               lowercase       ,    truncation=lowercase       ,    max_length=48_000       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (2, 1, 48_000)    )

           # pad to chunk
           UpperCAmelCase      	=		feature_extractor(lowercase       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(padded_outputs.shape       ,    (2, 1, 95_280)    )

           # pad to chunk
           UpperCAmelCase      	=		feature_extractor(lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (2, 1, 95_280)    )

           # force pad to max length
           UpperCAmelCase      	=		feature_extractor(
               lowercase       ,    padding='''max_length'''       ,    max_length=100_000       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (2, 1, 100_000)    )

           # force no pad
           with self.assertRaisesRegex(
               lowercase       ,    '''^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$'''       ,    ):
               UpperCAmelCase      	=		feature_extractor(lowercase       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values

           UpperCAmelCase      	=		feature_extractor(input_audio[0]       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (1, 1, 93_680)    )

           # no pad if no chunk_length_s
           UpperCAmelCase      	=		None
           with self.assertRaisesRegex(
               lowercase       ,    '''^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$'''       ,    ):
               UpperCAmelCase      	=		feature_extractor(lowercase       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values

           UpperCAmelCase      	=		feature_extractor(input_audio[0]       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (1, 1, 93_680)    )

           # no pad if no overlap
           UpperCAmelCase      	=		2
           UpperCAmelCase      	=		None
           with self.assertRaisesRegex(
               lowercase       ,    '''^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$'''       ,    ):
               UpperCAmelCase      	=		feature_extractor(lowercase       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values

           UpperCAmelCase      	=		feature_extractor(input_audio[0]       ,    padding=lowercase       ,    return_tensors='''pt'''    ).input_values
           self.assertEquals(truncated_outputs.shape       ,    (1, 1, 93_680)    )










'''simple docstring'''







import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase


@require_tokenizers
class       _a  (     unittest.TestCase			):
       __a    :					Optional[int]           = EsmTokenizer
       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           super().setUp()
           UpperCAmelCase      	=		tempfile.mkdtemp()
           # fmt: off
           UpperCAmelCase      	=		["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
           # fmt: on
           UpperCAmelCase      	=		os.path.join(self.tmpdirname       ,    VOCAB_FILES_NAMES['''vocab_file''']    )
           with open(self.vocab_file       ,    '''w'''       ,    encoding='''utf-8'''    ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
       def    A				(     self						:     Union[str, Any]       ,    **lowercase						:     Tuple    ):



           '''simple docstring'''

           return [self.get_tokenizer(**lowercase    )]
       def    A				(     self						:     Optional[Any]       ,    **lowercase						:     str    ):



           '''simple docstring'''

           return self.tokenizer_class.from_pretrained(self.tmpdirname       ,    **lowercase    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		tokenizer.tokenize('''LAGVS'''    )
           self.assertListEqual(lowercase       ,    ['''L''', '''A''', '''G''', '''V''', '''S''']    )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )       ,    [4, 5, 6, 7, 8]    )
       def    A				(     self						:     str    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		'''LAGVS'''
           self.assertListEqual(tokenizer.encode(lowercase    )       ,    [0, 4, 5, 6, 7, 8, 2]    )
       def    A				(     self						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		['''LAGVS''', '''WCB''']
           UpperCAmelCase      	=		tokenizer(lowercase       ,    padding=lowercase    )['''input_ids''']

           self.assertListEqual(lowercase       ,    [[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]    )
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.tokenizer_class(self.vocab_file    )

           UpperCAmelCase      	=		['''LAGVS''', '''WCB''']
           UpperCAmelCase      	=		tokenizer(lowercase       ,    padding=lowercase    )['''input_ids''']

           self.assertListEqual(lowercase       ,    [[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]    )







       def    A				(     self						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.get_tokenizers(fast=lowercase    )
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"    ):
                   UpperCAmelCase      	=		'''<unk>'''
                   UpperCAmelCase      	=		'''<mask>'''

                   UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )
                   UpperCAmelCase      	=		tokenizer.tokenize(lowercase    )

                   self.assertEqual(len(lowercase    )       ,    1    )
                   self.assertEqual(len(lowercase    )       ,    1    )
                   self.assertEqual(token_a[0]       ,    lowercase    )
                   self.assertEqual(token_a[0]       ,    lowercase    )




