



import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_UpperCAmelCase       =     logging.getLogger(__name__)





def        __magic_name__			(							):
   SCREAMING_SNAKE_CASE_:	Optional[int]            =argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""	)
   parser.add_argument(
       """--dataset_name"""							,     type=lowercase							,     default="""wikitext"""							,     help="""Name of the training. Explore datasets at: hf.co/datasets."""							,     )
   parser.add_argument(
       """--dataset_config"""							,     type=lowercase							,     default="""wikitext-103-raw-v1"""							,     help="""Configuration name of the dataset."""	)
   parser.add_argument(
       """--batch_size"""							,     type=lowercase							,     default=1000							,     help="""Batch size during training."""							,     )
   parser.add_argument(
       """--vocab_size"""							,     type=lowercase							,     default=1_0048							,     help="""Size of the desired vocabulary."""							,     )
   parser.add_argument(
       """--limit"""							,     default=lowercase							,     type=lowercase							,     help="""Limit the number of shards (used for debugging)."""							,     )
   parser.add_argument(
       """--export_to_hub"""							,     action="""store_true"""							,     )

   SCREAMING_SNAKE_CASE_:	List[Any]            =parser.parse_args()
   return args





def        __magic_name__			(							lowercase	):
   SCREAMING_SNAKE_CASE_:	List[str]            =datasets.load_dataset(args.dataset_name							,     args.dataset_config							,     split="""train"""	)

   if args.limit is not None:
      SCREAMING_SNAKE_CASE_:	Any            =min(len(lowercase	)							,     args.limit	)
      SCREAMING_SNAKE_CASE_:	str            =dataset.select(range(lowercase	)	)
      logger.info(f'''Limiting the dataset to {args.limit} entries.'''	)

   def batch_iterator():
      for i in range(0							,     len(lowercase	)							,     args.batch_size	):
         yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
   SCREAMING_SNAKE_CASE_:	Any            =Tokenizer(Unigram()	)
   SCREAMING_SNAKE_CASE_:	Dict            =normalizers.Sequence([normalizers.Replace("""``"""							,     """\""""	), normalizers.Replace("""''"""							,     """\""""	)]	)
   SCREAMING_SNAKE_CASE_:	Optional[int]            =pre_tokenizers.Metaspace()

   # Prepare the trainer.
   SCREAMING_SNAKE_CASE_:	Union[str, Any]            =UnigramTrainer(
       unk_token="""<unk>"""							,     special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]							,     vocab_size=args.vocab_size							,     )

   logger.info("""Training the tokenizer."""	)
   tokenizer.train_from_iterator(batch_iterator()							,     trainer=lowercase	)
   logger.info("""Tokenizer training complete!"""	)

   SCREAMING_SNAKE_CASE_:	List[str]            =tokenizer.token_to_id("""[CLS]"""	)
   SCREAMING_SNAKE_CASE_:	Union[str, Any]            =tokenizer.token_to_id("""[SEP]"""	)
   SCREAMING_SNAKE_CASE_:	Any            =processors.TemplateProcessing(
       single="""[CLS]:0 $A:0 [SEP]:0"""							,     pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""							,     special_tokens=[
           ("""[CLS]""", cls_token_id),
           ("""[SEP]""", sep_token_id),
       ]							,     )
   SCREAMING_SNAKE_CASE_:	Optional[int]            =decoders.Metaspace()

   if args.export_to_hub:
      logger.info("""Exporting the trained tokenzier to Hub."""	)
      SCREAMING_SNAKE_CASE_:	str            =AlbertTokenizerFast(tokenizer_object=lowercase	)
      new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""	)


if __name__ == "__main__":
   _UpperCAmelCase       =     parse_args()
   main(args)








"""simple docstring"""
def        __magic_name__			(							lowercase = 1000	):
   SCREAMING_SNAKE_CASE_:	Dict            =0
   SCREAMING_SNAKE_CASE_:	Any            =0
   while 1:
      num += 3
      if num >= n:
         break
      total += num
      num += 2
      if num >= n:
         break
      total += num
      num += 1
      if num >= n:
         break
      total += num
      num += 3
      if num >= n:
         break
      total += num
      num += 1
      if num >= n:
         break
      total += num
      num += 2
      if num >= n:
         break
      total += num
      num += 3
      if num >= n:
         break
      total += num
   return total


if __name__ == "__main__":
   print(f"""{solution() = }""")



