




"""simple docstring"""




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


a_            =   logging.getLogger(__name__)







def       __UpperCAmelCase      (				):
			__lowercase	:       Union[str, Any]				=   argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''					)
			parser.add_argument(
			    '''--dataset_name'''					,							type=__UpperCamelCase					,							default='''wikitext'''					,							help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,							)
			parser.add_argument(
			    '''--dataset_config'''					,							type=__UpperCamelCase					,							default='''wikitext-103-raw-v1'''					,							help='''Configuration name of the dataset.'''					)
			parser.add_argument(
			    '''--batch_size'''					,							type=__UpperCamelCase					,							default=10_00					,							help='''Batch size during training.'''					,							)
			parser.add_argument(
			    '''--vocab_size'''					,							type=__UpperCamelCase					,							default=1_00_48					,							help='''Size of the desired vocabulary.'''					,							)
			parser.add_argument(
			    '''--limit'''					,							default=__UpperCamelCase					,							type=__UpperCamelCase					,							help='''Limit the number of shards (used for debugging).'''					,							)
			parser.add_argument(
			    '''--export_to_hub'''					,							action='''store_true'''					,							)

			__lowercase	:       str				=   parser.parse_args()
			return args







def       __UpperCAmelCase      (				__UpperCamelCase					):
			__lowercase	:       Dict				=   datasets.load_dataset(args.dataset_name					,							args.dataset_config					,							split='''train'''					)

			if args.limit is not None:
						__lowercase	:       Tuple				=   min(len(__UpperCamelCase					)					,							args.limit					)
						__lowercase	:       Any				=   dataset.select(range(__UpperCamelCase					)					)
						logger.info(f"""Limiting the dataset to {args.limit} entries."""					)

			def batch_iterator():
						for i in range(0					,							len(__UpperCamelCase					)					,							args.batch_size					):
									yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
			__lowercase	:       Optional[Any]				=   Tokenizer(Unigram()					)
			__lowercase	:       Optional[int]				=   normalizers.Sequence([normalizers.Replace('''``'''					,							'''"'''					), normalizers.Replace('''\'\''''					,							'''"'''					)]					)
			__lowercase	:       List[Any]				=   pre_tokenizers.Metaspace()

			# Prepare the trainer.
			__lowercase	:       Any				=   UnigramTrainer(
			    unk_token='''<unk>'''					,							special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]''']					,							vocab_size=args.vocab_size					,							)

			logger.info('''Training the tokenizer.'''					)
			tokenizer.train_from_iterator(batch_iterator()					,							trainer=__UpperCamelCase					)
			logger.info('''Tokenizer training complete!'''					)

			__lowercase	:       Union[str, Any]				=   tokenizer.token_to_id('''[CLS]'''					)
			__lowercase	:       int				=   tokenizer.token_to_id('''[SEP]'''					)
			__lowercase	:       Optional[int]				=   processors.TemplateProcessing(
			    single='''[CLS]:0 $A:0 [SEP]:0'''					,							pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'''					,							special_tokens=[
			        ('''[CLS]''', cls_token_id),
			        ('''[SEP]''', sep_token_id),
			    ]					,							)
			__lowercase	:       Optional[Any]				=   decoders.Metaspace()

			if args.export_to_hub:
						logger.info('''Exporting the trained tokenzier to Hub.'''					)
						__lowercase	:       str				=   AlbertTokenizerFast(tokenizer_object=__UpperCamelCase					)
						new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''					)


if __name__ == "__main__":
			a_            =   parse_args()
			main(args)









"""simple docstring"""




import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
			from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

			from .test_feature_extraction_speechta import floats_list


a_            =   get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')

@require_torch
class        UpperCAmelCase_  (	unittest.TestCase						):
		def 					_lowerCamelCase    (		self   )		->   Dict:
					__lowercase	:       Union[str, Any]				=   tempfile.mkdtemp()

					__lowercase	:       List[str]				=   SpeechTaTokenizer(UpperCamelCase_   )
					tokenizer.save_pretrained(self.tmpdirname   )

					__lowercase	:       Dict				=   {
					    '''feature_size''': 1,
					    '''padding_value''': 0.0,
					    '''sampling_rate''': 1_60_00,
					    '''do_normalize''': False,
					    '''num_mel_bins''': 80,
					    '''hop_length''': 16,
					    '''win_length''': 64,
					    '''win_function''': '''hann_window''',
					    '''fmin''': 80,
					    '''fmax''': 76_00,
					    '''mel_floor''': 1E-10,
					    '''reduction_factor''': 2,
					    '''return_attention_mask''': True,
					}

					__lowercase	:       List[Any]				=   os.path.join(self.tmpdirname							,		UpperCamelCase_   )
					with open(self.feature_extraction_file							,		'''w'''							,		encoding='''utf-8'''   ) as fp:
								fp.write(json.dumps(UpperCamelCase_   ) + '''\n'''   )
		def 					_lowerCamelCase    (		self							,		**UpperCamelCase_   )		->   Union[str, Any]:
					return SpeechTaTokenizer.from_pretrained(self.tmpdirname							,		**UpperCamelCase_   )
		def 					_lowerCamelCase    (		self							,		**UpperCamelCase_   )		->   Optional[Any]:
					return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname							,		**UpperCamelCase_   )
		def 					_lowerCamelCase    (		self   )		->   Any:
					shutil.rmtree(self.tmpdirname   )
		def 					_lowerCamelCase    (		self   )		->   Union[str, Any]:
					__lowercase	:       List[str]				=   self.get_tokenizer()
					__lowercase	:       int				=   self.get_feature_extractor()

					__lowercase	:       Any				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					processor.save_pretrained(self.tmpdirname   )
					__lowercase	:       List[Any]				=   SpeechTaProcessor.from_pretrained(self.tmpdirname   )

					self.assertEqual(processor.tokenizer.get_vocab()							,		tokenizer.get_vocab()   )
					self.assertIsInstance(processor.tokenizer							,		UpperCamelCase_   )

					self.assertEqual(processor.feature_extractor.to_json_string()							,		feature_extractor.to_json_string()   )
					self.assertIsInstance(processor.feature_extractor							,		UpperCamelCase_   )
		def 					_lowerCamelCase    (		self   )		->   Optional[Any]:
					__lowercase	:       Optional[int]				=   SpeechTaProcessor(tokenizer=self.get_tokenizer()							,		feature_extractor=self.get_feature_extractor()   )
					processor.save_pretrained(self.tmpdirname   )

					__lowercase	:       str				=   self.get_tokenizer(bos_token='''(BOS)'''							,		eos_token='''(EOS)'''   )
					__lowercase	:       Dict				=   self.get_feature_extractor(do_normalize=UpperCamelCase_							,		padding_value=1.0   )

					__lowercase	:       Tuple				=   SpeechTaProcessor.from_pretrained(
					    self.tmpdirname							,		bos_token='''(BOS)'''							,		eos_token='''(EOS)'''							,		do_normalize=UpperCamelCase_							,		padding_value=1.0   )

					self.assertEqual(processor.tokenizer.get_vocab()							,		tokenizer_add_kwargs.get_vocab()   )
					self.assertIsInstance(processor.tokenizer							,		UpperCamelCase_   )

					self.assertEqual(processor.feature_extractor.to_json_string()							,		feature_extractor_add_kwargs.to_json_string()   )
					self.assertIsInstance(processor.feature_extractor							,		UpperCamelCase_   )
		def 					_lowerCamelCase    (		self   )		->   List[str]:
					__lowercase	:       Optional[Any]				=   self.get_feature_extractor()
					__lowercase	:       Optional[Any]				=   self.get_tokenizer()

					__lowercase	:       Union[str, Any]				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					__lowercase	:       int				=   floats_list((3, 10_00)   )

					__lowercase	:       str				=   feature_extractor(audio=UpperCamelCase_							,		return_tensors='''np'''   )
					__lowercase	:       Dict				=   processor(audio=UpperCamelCase_							,		return_tensors='''np'''   )

					for key in input_feat_extract.keys():
								self.assertAlmostEqual(input_feat_extract[key].sum()							,		input_processor[key].sum()							,		delta=1E-2   )
		def 					_lowerCamelCase    (		self   )		->   List[Any]:
					__lowercase	:       str				=   self.get_feature_extractor()
					__lowercase	:       str				=   self.get_tokenizer()

					__lowercase	:       Union[str, Any]				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					__lowercase	:       Optional[int]				=   floats_list((3, 10_00)   )

					__lowercase	:       List[str]				=   feature_extractor(audio_target=UpperCamelCase_							,		return_tensors='''np'''   )
					__lowercase	:       List[str]				=   processor(audio_target=UpperCamelCase_							,		return_tensors='''np'''   )

					for key in input_feat_extract.keys():
								self.assertAlmostEqual(input_feat_extract[key].sum()							,		input_processor[key].sum()							,		delta=1E-2   )
		def 					_lowerCamelCase    (		self   )		->   Any:
					__lowercase	:       List[Any]				=   self.get_feature_extractor()
					__lowercase	:       List[str]				=   self.get_tokenizer()

					__lowercase	:       Union[str, Any]				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					__lowercase	:       List[str]				=   '''This is a test string'''

					__lowercase	:       Any				=   processor(text=UpperCamelCase_   )
					__lowercase	:       Optional[int]				=   tokenizer(UpperCamelCase_   )

					for key in encoded_tok.keys():
								self.assertListEqual(encoded_tok[key]							,		encoded_processor[key]   )
		def 					_lowerCamelCase    (		self   )		->   str:
					__lowercase	:       int				=   self.get_feature_extractor()
					__lowercase	:       List[str]				=   self.get_tokenizer()

					__lowercase	:       List[Any]				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					__lowercase	:       Union[str, Any]				=   '''This is a test string'''

					__lowercase	:       Tuple				=   processor(text_target=UpperCamelCase_   )
					__lowercase	:       Optional[int]				=   tokenizer(UpperCamelCase_   )

					for key in encoded_tok.keys():
								self.assertListEqual(encoded_tok[key]							,		encoded_processor[key]   )
		def 					_lowerCamelCase    (		self   )		->   Optional[Any]:
					__lowercase	:       Optional[int]				=   self.get_feature_extractor()
					__lowercase	:       str				=   self.get_tokenizer()

					__lowercase	:       int				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					__lowercase	:       Optional[Any]				=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

					__lowercase	:       str				=   processor.batch_decode(UpperCamelCase_   )
					__lowercase	:       int				=   tokenizer.batch_decode(UpperCamelCase_   )

					self.assertListEqual(UpperCamelCase_							,		UpperCamelCase_   )
		def 					_lowerCamelCase    (		self   )		->   Optional[int]:
					__lowercase	:       Union[str, Any]				=   self.get_feature_extractor()
					__lowercase	:       int				=   self.get_tokenizer()

					__lowercase	:       Optional[int]				=   SpeechTaProcessor(tokenizer=UpperCamelCase_							,		feature_extractor=UpperCamelCase_   )

					self.assertListEqual(
					    processor.model_input_names							,		feature_extractor.model_input_names							,		msg='''`processor` and `feature_extractor` model input names do not match'''							,		)



