





"""simple docstring"""





import argparse
import os





class       snake_case		:
      lowerCamelCase__      =			'''block_{0}.dat'''

      def __init__(      self       :int						,							_lowerCamelCase       :Dict						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	filename
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	[]

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						,							_lowerCamelCase       :Tuple						,							_lowerCamelCase       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.BLOCK_FILENAME_FORMAT.format(_lowerCamelCase						)
       with open(_lowerCamelCase						,							'''w'''						) as file:
        file.write(_lowerCamelCase						)
       self.block_filenames.append(_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :int						):
       return self.block_filenames

      def 						SCREAMING_SNAKE_CASE_      (      self       :Tuple						,							_lowerCamelCase       :List[str]						,							_lowerCamelCase       :Optional[Any]=None						):
       __SCREAMING_SNAKE_CASE			:      int 							=	0
       with open(self.filename						) as file:
        while True:
         __SCREAMING_SNAKE_CASE			:      Any 							=	file.readlines(_lowerCamelCase						)

         if lines == []:
          break

         if sort_key is None:
          lines.sort()
         else:
          lines.sort(key=_lowerCamelCase						)

         self.write_block(''''''.join(_lowerCamelCase						)						,							_lowerCamelCase						)
         i += 1



      def 						SCREAMING_SNAKE_CASE_      (      self       :Tuple						):
       map(os.remove						,							self.block_filenames						)







class       snake_case		:

      def 						SCREAMING_SNAKE_CASE_      (      self       :int						,							_lowerCamelCase       :Tuple						):
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	-1
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	None

       for i in range(len(_lowerCamelCase						)						):
        if min_str is None or choices[i] < min_str:
         __SCREAMING_SNAKE_CASE			:      Tuple 							=	i

       return min_index







class       snake_case		:

      def __init__(      self       :str						,							_lowerCamelCase       :Tuple						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	files
       __SCREAMING_SNAKE_CASE			:      str 							=	set()
       __SCREAMING_SNAKE_CASE			:      Dict 							=	len(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      str 							=	{i: None for i in range(self.num_buffers						)}

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       return {
           i: self.buffers[i] for i in range(self.num_buffers						) if i not in self.empty
       }

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       for i in range(self.num_buffers						):
        if self.buffers[i] is None and i not in self.empty:
         __SCREAMING_SNAKE_CASE			:      int 							=	self.files[i].readline()

         if self.buffers[i] == "":
          self.empty.add(_lowerCamelCase						)
          self.files[i].close()

       if len(self.empty						) == self.num_buffers:
        return False

       return True



      def 						SCREAMING_SNAKE_CASE_      (      self       :Any						,							_lowerCamelCase       :List[str]						):
       __SCREAMING_SNAKE_CASE			:      Dict 							=	self.buffers[index]
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	None

       return value







class       snake_case		:

      def __init__(      self       :List[Any]						,							_lowerCamelCase       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	merge_strategy

      def 						SCREAMING_SNAKE_CASE_      (      self       :int						,							_lowerCamelCase       :Tuple						,							_lowerCamelCase       :Any						,							_lowerCamelCase       :str						):
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	FilesArray(self.get_file_handles(_lowerCamelCase						,							_lowerCamelCase						)						)
       with open(_lowerCamelCase						,							'''w'''						,							_lowerCamelCase						) as outfile:
        while buffers.refresh():
         __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.merge_strategy.select(buffers.get_dict()						)
         outfile.write(buffers.unshift(_lowerCamelCase						)						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						,							_lowerCamelCase       :int						,							_lowerCamelCase       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	{}

       for i in range(len(_lowerCamelCase						)						):
        __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	open(filenames[i]						,							'''r'''						,							_lowerCamelCase						)  # noqa: UP015

       return files







class       snake_case		:

      def __init__(      self       :Dict						,							_lowerCamelCase       :str						):
       __SCREAMING_SNAKE_CASE			:      int 							=	block_size

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						,							_lowerCamelCase       :List[str]						,							_lowerCamelCase       :Dict=None						):
       __SCREAMING_SNAKE_CASE			:      Dict 							=	self.get_number_blocks(_lowerCamelCase						,							self.block_size						)
       __SCREAMING_SNAKE_CASE			:      Dict 							=	FileSplitter(_lowerCamelCase						)
       splitter.split(self.block_size						,							_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Any 							=	FileMerger(NWayMerge()						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	self.block_size / (num_blocks + 1)
       merger.merge(splitter.get_block_filenames()						,							filename + '''.out'''						,							_lowerCamelCase						)

       splitter.cleanup()



      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						,							_lowerCamelCase       :str						,							_lowerCamelCase       :List[Any]						):
       return (os.stat(_lowerCamelCase						).st_size / block_size) + 1







def 		lowerCAmelCase_     (  lowercase_       :							str		):

 '''simple docstring'''






 if string[-1].lower() == "k":
  return int(string[:-1]		) * 1024
 elif string[-1].lower() == "m":
  return int(string[:-1]		) * 1024 * 1024
 elif string[-1].lower() == "g":
  return int(string[:-1]		) * 1024 * 1024 * 1024
 else:
  return int(lowercase_		)




def 		lowerCAmelCase_     (  ):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	argparse.ArgumentParser()
 parser.add_argument(
     '''-m'''			,				'''--mem'''			,				help='''amount of memory to use for sorting'''			,				default='''100M'''		)
 parser.add_argument(
     '''filename'''			,				metavar='''<filename>'''			,				nargs=1			,				help='''name of file to sort'''		)
 __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	parser.parse_args()

 __SCREAMING_SNAKE_CASE			:      List[str] 							=	ExternalSort(parse_memory(args.mem		)		)
 sorter.sort(args.filename[0]		)


if __name__ == "__main__":
    main()








"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
    from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

    from .test_feature_extraction_speechta import floats_list


_lowerCamelCase  					=     get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')





@require_torch
class       snake_case		(       unittest.TestCase       ):

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	tempfile.mkdtemp()

       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	SpeechTaTokenizer(_lowerCamelCase						)
       tokenizer.save_pretrained(self.tmpdirname						)

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	{
           '''feature_size''': 1,
           '''padding_value''': 0.0,
           '''sampling_rate''': 1_6_0_0_0,
           '''do_normalize''': False,
           '''num_mel_bins''': 8_0,
           '''hop_length''': 1_6,
           '''win_length''': 6_4,
           '''win_function''': '''hann_window''',
           '''fmin''': 8_0,
           '''fmax''': 7_6_0_0,
           '''mel_floor''': 1e-10,
           '''reduction_factor''': 2,
           '''return_attention_mask''': True,
       }

       __SCREAMING_SNAKE_CASE			:      Any 							=	os.path.join(self.tmpdirname						,							_lowerCamelCase						)
       with open(self.feature_extraction_file						,							'''w'''						,							encoding='''utf-8'''						) as fp:
        fp.write(json.dumps(_lowerCamelCase						) + '''\n'''						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						,							**_lowerCamelCase       :str						):
       return SpeechTaTokenizer.from_pretrained(self.tmpdirname						,							**_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Dict						,							**_lowerCamelCase       :Dict						):
       return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname						,							**_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[str]						):
       shutil.rmtree(self.tmpdirname						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.get_tokenizer()
       __SCREAMING_SNAKE_CASE			:      int 							=	self.get_feature_extractor()

       __SCREAMING_SNAKE_CASE			:      Any 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       processor.save_pretrained(self.tmpdirname						)
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	SpeechTaProcessor.from_pretrained(self.tmpdirname						)

       self.assertEqual(processor.tokenizer.get_vocab()						,							tokenizer.get_vocab()						)
       self.assertIsInstance(processor.tokenizer						,							_lowerCamelCase						)

       self.assertEqual(processor.feature_extractor.to_json_string()						,							feature_extractor.to_json_string()						)
       self.assertIsInstance(processor.feature_extractor						,							_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Union[str, Any]						):
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	SpeechTaProcessor(tokenizer=self.get_tokenizer()						,							feature_extractor=self.get_feature_extractor()						)
       processor.save_pretrained(self.tmpdirname						)

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.get_tokenizer(bos_token='''(BOS)'''						,							eos_token='''(EOS)'''						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	self.get_feature_extractor(do_normalize=_lowerCamelCase						,							padding_value=1.0						)

       __SCREAMING_SNAKE_CASE			:      Any 							=	SpeechTaProcessor.from_pretrained(
           self.tmpdirname						,							bos_token='''(BOS)'''						,							eos_token='''(EOS)'''						,							do_normalize=_lowerCamelCase						,							padding_value=1.0						)

       self.assertEqual(processor.tokenizer.get_vocab()						,							tokenizer_add_kwargs.get_vocab()						)
       self.assertIsInstance(processor.tokenizer						,							_lowerCamelCase						)

       self.assertEqual(processor.feature_extractor.to_json_string()						,							feature_extractor_add_kwargs.to_json_string()						)
       self.assertIsInstance(processor.feature_extractor						,							_lowerCamelCase						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :str						):
       __SCREAMING_SNAKE_CASE			:      Optional[int] 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      int 							=	floats_list((3, 1_0_0_0)						)

       __SCREAMING_SNAKE_CASE			:      int 							=	feature_extractor(audio=_lowerCamelCase						,							return_tensors='''np'''						)
       __SCREAMING_SNAKE_CASE			:      List[str] 							=	processor(audio=_lowerCamelCase						,							return_tensors='''np'''						)

       for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()						,							input_processor[key].sum()						,							delta=1e-2						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Tuple						):
       __SCREAMING_SNAKE_CASE			:      Any 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      int 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Any 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	floats_list((3, 1_0_0_0)						)

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	feature_extractor(audio_target=_lowerCamelCase						,							return_tensors='''np'''						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	processor(audio_target=_lowerCamelCase						,							return_tensors='''np'''						)

       for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()						,							input_processor[key].sum()						,							delta=1e-2						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :List[Any]						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      int 							=	'''This is a test string'''

       __SCREAMING_SNAKE_CASE			:      str 							=	processor(text=_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Union[str, Any] 							=	tokenizer(_lowerCamelCase						)

       for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]						,							encoded_processor[key]						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[int]						):
       __SCREAMING_SNAKE_CASE			:      int 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      str 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Tuple 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Dict 							=	'''This is a test string'''

       __SCREAMING_SNAKE_CASE			:      str 							=	processor(text_target=_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      Dict 							=	tokenizer(_lowerCamelCase						)

       for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]						,							encoded_processor[key]						)

      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Dict 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      Tuple 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	processor.batch_decode(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE			:      List[Any] 							=	tokenizer.batch_decode(_lowerCamelCase						)

       self.assertListEqual(_lowerCamelCase						,							_lowerCamelCase						)



      def 						SCREAMING_SNAKE_CASE_      (      self       :Optional[Any]						):
       __SCREAMING_SNAKE_CASE			:      Optional[Any] 							=	self.get_feature_extractor()
       __SCREAMING_SNAKE_CASE			:      str 							=	self.get_tokenizer()

       __SCREAMING_SNAKE_CASE			:      str 							=	SpeechTaProcessor(tokenizer=_lowerCamelCase						,							feature_extractor=_lowerCamelCase						)

       self.assertListEqual(
           processor.model_input_names						,							feature_extractor.model_input_names						,							msg='''`processor` and `feature_extractor` model input names do not match'''						,							)

