



import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp

def      A_ (		):

				"""simple docstring"""
				a__			 =						ArgumentParser(
				    description=(
				        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
				    )						)

				# Optional arguments for the launch helper
				parser.add_argument("""--num_cores"""	,						type=__a	,						default=1	,						help="""Number of TPU cores to use (1 or 8)."""						)

				# positional
				parser.add_argument(
				    """training_script"""	,						type=__a	,						help=(
				        """The full path to the single TPU training """
				        """program/script to be launched in parallel, """
				        """followed by all the arguments for the """
				        """training script"""
				    )	,						)

				# rest from the training program
				parser.add_argument("""training_script_args"""	,						nargs=__a						)

				return parser.parse_args()

def      A_ (		):

				"""simple docstring"""
				a__			 =						parse_args()

				# Import training_script as a module.
				a__			 =						Path(args.training_script						)
				sys.path.append(str(script_fpath.parent.resolve()						)						)
				a__			 =						script_fpath.stem
				a__			 =						importlib.import_module(__a						)

				# Patch sys.argv
				a__			 =						[args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores						)]

				xmp.spawn(mod._mp_fn	,						args=()	,						nprocs=args.num_cores						)


if __name__ == "__main__":
	main()






import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
	from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

	from .test_feature_extraction_speechta import floats_list


UpperCAmelCase								=   get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")

@require_torch
class 	__snake_case (				unittest.TestCase):







							'''simple docstring'''






							def _a (							self						):
											a__			 =						tempfile.mkdtemp()

											a__			 =						SpeechTaTokenizer(a_						)
											tokenizer.save_pretrained(self.tmpdirname						)

											a__			 =						{
											    """feature_size""": 1,
											    """padding_value""": 0.0,
											    """sampling_rate""": 16_000,
											    """do_normalize""": False,
											    """num_mel_bins""": 80,
											    """hop_length""": 16,
											    """win_length""": 64,
											    """win_function""": """hann_window""",
											    """fmin""": 80,
											    """fmax""": 7_600,
											    """mel_floor""": 1E-1_0,
											    """reduction_factor""": 2,
											    """return_attention_mask""": True,
											}

											a__			 =						os.path.join(self.tmpdirname					,					a_						)
											with open(self.feature_extraction_file					,					"""w"""					,					encoding="""utf-8"""						) as fp:
															fp.write(json.dumps(a_						) + """\n"""						)


							def _a (							self					,					**a_						):
											return SpeechTaTokenizer.from_pretrained(self.tmpdirname					,					**a_						)


							def _a (							self					,					**a_						):
											return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname					,					**a_						)


							def _a (							self						):
											shutil.rmtree(self.tmpdirname						)


							def _a (							self						):
											a__			 =						self.get_tokenizer()
											a__			 =						self.get_feature_extractor()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											processor.save_pretrained(self.tmpdirname						)
											a__			 =						SpeechTaProcessor.from_pretrained(self.tmpdirname						)

											self.assertEqual(processor.tokenizer.get_vocab()					,					tokenizer.get_vocab()						)
											self.assertIsInstance(processor.tokenizer					,					a_						)

											self.assertEqual(processor.feature_extractor.to_json_string()					,					feature_extractor.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor					,					a_						)


							def _a (							self						):
											a__			 =						SpeechTaProcessor(tokenizer=self.get_tokenizer()					,					feature_extractor=self.get_feature_extractor()						)
											processor.save_pretrained(self.tmpdirname						)

											a__			 =						self.get_tokenizer(bos_token="""(BOS)"""					,					eos_token="""(EOS)"""						)
											a__			 =						self.get_feature_extractor(do_normalize=a_					,					padding_value=1.0						)

											a__			 =						SpeechTaProcessor.from_pretrained(
											    self.tmpdirname					,					bos_token="""(BOS)"""					,					eos_token="""(EOS)"""					,					do_normalize=a_					,					padding_value=1.0						)

											self.assertEqual(processor.tokenizer.get_vocab()					,					tokenizer_add_kwargs.get_vocab()						)
											self.assertIsInstance(processor.tokenizer					,					a_						)

											self.assertEqual(processor.feature_extractor.to_json_string()					,					feature_extractor_add_kwargs.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor					,					a_						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						floats_list((3, 1_000)						)

											a__			 =						feature_extractor(audio=a_					,					return_tensors="""np"""						)
											a__			 =						processor(audio=a_					,					return_tensors="""np"""						)

											for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()					,					input_processor[key].sum()					,					delta=1E-2						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						floats_list((3, 1_000)						)

											a__			 =						feature_extractor(audio_target=a_					,					return_tensors="""np"""						)
											a__			 =						processor(audio_target=a_					,					return_tensors="""np"""						)

											for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()					,					input_processor[key].sum()					,					delta=1E-2						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						"""This is a test string"""

											a__			 =						processor(text=a_						)
											a__			 =						tokenizer(a_						)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]					,					encoded_processor[key]						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						"""This is a test string"""

											a__			 =						processor(text_target=a_						)
											a__			 =						tokenizer(a_						)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]					,					encoded_processor[key]						)


							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											a__			 =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											a__			 =						processor.batch_decode(a_						)
											a__			 =						tokenizer.batch_decode(a_						)

											self.assertListEqual(a_					,					a_						)







							def _a (							self						):
											a__			 =						self.get_feature_extractor()
											a__			 =						self.get_tokenizer()

											a__			 =						SpeechTaProcessor(tokenizer=a_					,					feature_extractor=a_						)

											self.assertListEqual(
											    processor.model_input_names					,					feature_extractor.model_input_names					,					msg="""`processor` and `feature_extractor` model input names do not match"""					,					)

