






from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


__UpperCamelCase   :						Optional[Any]							=  logging.get_logger(__name__)


class 					__SCREAMING_SNAKE_CASE(		_UpperCAmelCase		):
							_UpperCAmelCase	      =    ["input_values", "attention_mask"]
							def __init__(       self: Tuple    ,		UpperCamelCase: Union[str, Any]=1    ,		UpperCamelCase: Optional[Any]=1_60_00    ,		UpperCamelCase: str=1_28    ,		UpperCamelCase: int=10_24    ,		UpperCamelCase: List[str]=0.0    ,		UpperCamelCase: Any=True    ,		UpperCamelCase: str=-4.2_677_393    ,		UpperCamelCase: List[Any]=4.5_689_974    ,		UpperCamelCase: Dict=False    ,		**UpperCamelCase: str    ,		)						->						List[Any]:
											super().__init__(feature_size=lowercase__    ,		sampling_rate=lowercase__    ,		padding_value=lowercase__    ,		**lowercase__  )
											snake_case__													=       num_mel_bins
											snake_case__													=       max_length
											snake_case__													=       do_normalize
											snake_case__													=       mean
											snake_case__													=       std
											snake_case__													=       return_attention_mask
							def lowerCAmelCase_	(       self: int    ,		UpperCamelCase: Dict    ,		UpperCamelCase: int    ,		)						->						Optional[Any]:
											snake_case__													=       torch.from_numpy(lowercase__  ).unsqueeze(0  )
											snake_case__													=       ta_kaldi.fbank(
											    lowercase__    ,		htk_compat=lowercase__    ,		sample_frequency=self.sampling_rate    ,		use_energy=lowercase__    ,		window_type='hanning'    ,		num_mel_bins=self.num_mel_bins    ,		dither=0.0    ,		frame_shift=10    ,		)

											snake_case__													=       fbank.shape[0]
											snake_case__													=       max_length - n_frames

											# pad or truncate, depending on difference
											if difference > 0:
															snake_case__													=       torch.nn.ZeroPadad((0, 0, 0, difference)  )
															snake_case__													=       pad_module(lowercase__  )
											elif difference < 0:
															snake_case__													=       fbank[0:max_length, :]

											snake_case__													=       fbank.numpy()

											return fbank
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: Optional[int]  )						->						int:
											return (input_values - (self.mean)) / (self.std * 2)


							def __call__(       self: List[str]    ,		UpperCamelCase: Any    ,		UpperCamelCase: str = None    ,		UpperCamelCase: Optional[Any] = None    ,		**UpperCamelCase: Tuple    ,		)						->						List[str]:






											if sampling_rate is not None:
															if sampling_rate != self.sampling_rate:
																			raise ValueError(
																			    F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
																			    F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
																			    F''' {self.sampling_rate} and not {sampling_rate}.'''  )
											else:
															logger.warning(
															    'It is strongly recommended to pass the `sampling_rate` argument to this function. '
															    'Failing to do so can result in silent errors that might be hard to debug.'  )

											snake_case__													=       isinstance(lowercase__    ,		np.ndarray  ) and len(raw_speech.shape  ) > 1
											if is_batched_numpy and len(raw_speech.shape  ) > 2:
															raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''  )
											snake_case__													=       is_batched_numpy or (
											    isinstance(lowercase__    ,		(list, tuple)  ) and (isinstance(raw_speech[0]    ,		(np.ndarray, tuple, list)  ))
											)

											if is_batched:
															snake_case__													=       [np.asarray(lowercase__    ,		dtype=np.floataa  ) for speech in raw_speech]
											elif not is_batched and not isinstance(lowercase__    ,		np.ndarray  ):
															snake_case__													=       np.asarray(lowercase__    ,		dtype=np.floataa  )
											elif isinstance(lowercase__    ,		np.ndarray  ) and raw_speech.dtype is np.dtype(np.floataa  ):
															snake_case__													=       raw_speech.astype(np.floataa  )

											# always return batch
											if not is_batched:
															snake_case__													=       [raw_speech]

											# extract fbank features and pad/truncate to max_length
											snake_case__													=       [self._extract_fbank_features(lowercase__    ,		max_length=self.max_length  ) for waveform in raw_speech]

											# convert into BatchFeature
											snake_case__													=       BatchFeature({'input_values': features}  )

											# make sure list is in array format
											snake_case__													=       padded_inputs.get('input_values'  )
											if isinstance(input_values[0]    ,		lowercase__  ):
															snake_case__													=       [np.asarray(lowercase__    ,		dtype=np.floataa  ) for feature in input_values]

											# normalization
											if self.do_normalize:
															snake_case__													=       [self.normalize(lowercase__  ) for feature in input_values]

											if return_tensors is not None:
															snake_case__													=       padded_inputs.convert_to_tensors(lowercase__  )

											return padded_inputs












import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    ElectraTokenizer
							_UpperCAmelCase	      =    ElectraTokenizerFast
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    True
							_UpperCAmelCase	      =    filter_non_english
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											super().setUp()

											snake_case__													=       [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case__													=       os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file    ,		'w'    ,		encoding='utf-8'  ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
							def lowerCAmelCase_	(       self: str    ,		UpperCamelCase: List[str]  )						->						Union[str, Any]:
											snake_case__													=       'UNwant\u00E9d,running'
											snake_case__													=       'unwanted, running'
											return input_text, output_text
							def lowerCAmelCase_	(       self: List[str]  )						->						List[Any]:
											snake_case__													=       self.tokenizer_class(self.vocab_file  )

											snake_case__													=       tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(UpperCamelCase    ,		['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase  )    ,		[9, 6, 7, 12, 10, 11]  )
							def lowerCAmelCase_	(       self: str  )						->						Tuple:
											if not self.test_rust_tokenizer:
															return

											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											# With lower casing
											snake_case__													=       self.get_tokenizer(do_lower_case=UpperCamelCase  )
											snake_case__													=       self.get_rust_tokenizer(do_lower_case=UpperCamelCase  )

											snake_case__													=       'UNwant\u00E9d,running'

											snake_case__													=       tokenizer.tokenize(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.tokenize(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

											snake_case__													=       self.get_rust_tokenizer()
											snake_case__													=       tokenizer.encode(UpperCamelCase  )
											snake_case__													=       rust_tokenizer.encode(UpperCamelCase  )
											self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						int:
											snake_case__													=       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )    ,		['ah', '\u535A', '\u63A8', 'zz']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[Any]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['h\u00E9llo']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: str  )						->						Optional[int]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,		['hello']  )
							def lowerCAmelCase_	(       self: Any  )						->						int:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: str  )						->						List[str]:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: int  )						->						str:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		strip_accents=UpperCamelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       BasicTokenizer(do_lower_case=UpperCamelCase    ,		never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )    ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
							def lowerCAmelCase_	(       self: Any  )						->						Dict:
											snake_case__													=       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											snake_case__													=       {}
											for i, token in enumerate(UpperCamelCase  ):
															snake_case__													=       i
											snake_case__													=       WordpieceTokenizer(vocab=UpperCamelCase    ,		unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )    ,		[]  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )    ,		['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )    ,		['[UNK]', 'runn', '##ing']  )
							def lowerCAmelCase_	(       self: Dict  )						->						str:
											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
							def lowerCAmelCase_	(       self: List[str]  )						->						Tuple:
											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
							def lowerCAmelCase_	(       self: str  )						->						Any:
											snake_case__													=       self.get_tokenizer()
											snake_case__													=       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(UpperCamelCase  ) for t in ['Test', '\xad', 'test']]    ,		[['[UNK]'], [], ['[UNK]']]  )
							@slow
							def lowerCAmelCase_	(       self: Optional[int]  )						->						Tuple:
											snake_case__													=       self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											snake_case__													=       tokenizer.encode('sequence builders'    ,		add_special_tokens=UpperCamelCase  )
											snake_case__													=       tokenizer.encode('multi-sequence build'    ,		add_special_tokens=UpperCamelCase  )

											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase  )
											snake_case__													=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase    ,		UpperCamelCase  )

											assert encoded_sentence == [1_01] + text + [1_02]
											assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
							def lowerCAmelCase_	(       self: List[str]  )						->						Dict:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			snake_case__													=       tokenizer_r.encode_plus(
																			    UpperCamelCase    ,		return_attention_mask=UpperCamelCase    ,		return_token_type_ids=UpperCamelCase    ,		return_offsets_mapping=UpperCamelCase    ,		add_special_tokens=UpperCamelCase    ,		)

																			snake_case__													=       tokenizer_r.do_lower_case if hasattr(UpperCamelCase    ,		'do_lower_case'  ) else False
																			snake_case__													=       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]    ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																			self.assertEqual([e[0] for e in expected_results]    ,		tokens['offset_mapping']  )


							def lowerCAmelCase_	(       self: Tuple  )						->						Any:
											snake_case__													=       ['的', '人', '有']
											snake_case__													=       ''.join(UpperCamelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																			snake_case__													=       True
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )

																			snake_case__													=       False
																			snake_case__													=       self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )
																			snake_case__													=       self.tokenizer_class.from_pretrained(UpperCamelCase    ,		**UpperCamelCase  )

																			snake_case__													=       tokenizer_r.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )
																			snake_case__													=       tokenizer_p.encode(UpperCamelCase    ,		add_special_tokens=UpperCamelCase  )

																			snake_case__													=       tokenizer_r.convert_ids_to_tokens(UpperCamelCase  )
																			snake_case__													=       tokenizer_p.convert_ids_to_tokens(UpperCamelCase  )

																			# it is expected that only the first Chinese character is not preceded by "##".
																			snake_case__													=       [
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase  )
																			]
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )
																			self.assertListEqual(UpperCamelCase    ,		UpperCamelCase  )




