



'''simple docstring'''





import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


A			:   Dict			=      logging.getLogger(__name__)
def 		_a   (  ):
   snake_case   :							Optional[Any]						  =argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''							)
   parser.add_argument(
       '''--dataset_name'''					,						type=lowerCamelCase_					,						default='''wikitext'''					,						help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,						)
   parser.add_argument(
       '''--dataset_config'''					,						type=lowerCamelCase_					,						default='''wikitext-103-raw-v1'''					,						help='''Configuration name of the dataset.'''							)
   parser.add_argument(
       '''--batch_size'''					,						type=lowerCamelCase_					,						default=10_00					,						help='''Batch size during training.'''					,						)
   parser.add_argument(
       '''--vocab_size'''					,						type=lowerCamelCase_					,						default=1_00_48					,						help='''Size of the desired vocabulary.'''					,						)
   parser.add_argument(
       '''--limit'''					,						default=lowerCamelCase_					,						type=lowerCamelCase_					,						help='''Limit the number of shards (used for debugging).'''					,						)
   parser.add_argument(
       '''--export_to_hub'''					,						action='''store_true'''					,						)

   snake_case   :							Any						  =parser.parse_args()
   return args
def 		_a   (  lowerCamelCase_							):
   snake_case   :							int						  =datasets.load_dataset(args.dataset_name					,						args.dataset_config					,						split='''train'''							)

   if args.limit is not None:
      snake_case   :							List[Any]						  =min(len(lowerCamelCase_							)					,						args.limit							)
      snake_case   :							List[Any]						  =dataset.select(range(lowerCamelCase_							)							)
      logger.info(F'''Limiting the dataset to {args.limit} entries.'''							)

   def batch_iterator():
      for i in range(0					,						len(lowerCamelCase_							)					,						args.batch_size							):
         yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
   snake_case   :							Union[str, Any]						  =Tokenizer(Unigram()							)
   snake_case   :							int						  =normalizers.Sequence([normalizers.Replace('''``'''					,						'''"'''							), normalizers.Replace('''\'\''''					,						'''"'''							)]							)
   snake_case   :							Optional[Any]						  =pre_tokenizers.Metaspace()

   # Prepare the trainer.
   snake_case   :							str						  =UnigramTrainer(
       unk_token='''<unk>'''					,						special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]''']					,						vocab_size=args.vocab_size					,						)

   logger.info('''Training the tokenizer.'''							)
   tokenizer.train_from_iterator(batch_iterator()					,						trainer=lowerCamelCase_							)
   logger.info('''Tokenizer training complete!'''							)

   snake_case   :							Union[str, Any]						  =tokenizer.token_to_id('''[CLS]'''							)
   snake_case   :							Optional[Any]						  =tokenizer.token_to_id('''[SEP]'''							)
   snake_case   :							Optional[int]						  =processors.TemplateProcessing(
       single='''[CLS]:0 $A:0 [SEP]:0'''					,						pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'''					,						special_tokens=[
           ('''[CLS]''', cls_token_id),
           ('''[SEP]''', sep_token_id),
       ]					,						)
   snake_case   :							Tuple						  =decoders.Metaspace()

   if args.export_to_hub:
      logger.info('''Exporting the trained tokenzier to Hub.'''							)
      snake_case   :							Tuple						  =AlbertTokenizerFast(tokenizer_object=lowerCamelCase_							)
      new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''							)


if __name__ == "__main__":
      A			:   Dict			=      parse_args()
      main(args)










'''simple docstring'''





def 		_a   (  lowerCamelCase_							):
   return "".join(chr(ord(lowerCamelCase_							) + 32							) if '''A''' <= char <= '''Z''' else char for char in word							)


if __name__ == "__main__":
      from doctest import testmod

      testmod()





