





import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


A__:  int       					=					logging.getLogger(__name__)



def 							lowerCAmelCase_				(    ):
    UpperCamelCase__:					Union[str, Any]												=      argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset.")
    parser.add_argument(
        "--dataset_name"  ,type=A_  ,default="wikitext"  ,help="Name of the training. Explore datasets at: hf.co/datasets."  ,)
    parser.add_argument(
        "--dataset_config"  ,type=A_  ,default="wikitext-103-raw-v1"  ,help="Configuration name of the dataset.")
    parser.add_argument(
        "--batch_size"  ,type=A_  ,default=10_00  ,help="Batch size during training."  ,)
    parser.add_argument(
        "--vocab_size"  ,type=A_  ,default=1_00_48  ,help="Size of the desired vocabulary."  ,)
    parser.add_argument(
        "--limit"  ,default=A_  ,type=A_  ,help="Limit the number of shards (used for debugging)."  ,)
    parser.add_argument(
        "--export_to_hub"  ,action="store_true"  ,)

    UpperCamelCase__:					Tuple												=      parser.parse_args()
    return args



def 							lowerCAmelCase_				(    A_):
    UpperCamelCase__:					Dict												=      datasets.load_dataset(args.dataset_name  ,args.dataset_config  ,split="train")

    if args.limit is not None:
        UpperCamelCase__:					Optional[Any]												=      min(len(A_)  ,args.limit)
        UpperCamelCase__:					List[Any]												=      dataset.select(range(A_))
        logger.info(F"Limiting the dataset to {args.limit} entries.")

    def batch_iterator():
        for i in range(0  ,len(A_)  ,args.batch_size):
            yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
    UpperCamelCase__:					Union[str, Any]												=      Tokenizer(Unigram())
    UpperCamelCase__:					List[str]												=      normalizers.Sequence([normalizers.Replace("``"  ,"\""), normalizers.Replace("''"  ,"\"")])
    UpperCamelCase__:					Optional[Any]												=      pre_tokenizers.Metaspace()

    # Prepare the trainer.
    UpperCamelCase__:					List[str]												=      UnigramTrainer(
        unk_token="<unk>"  ,special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]  ,vocab_size=args.vocab_size  ,)

    logger.info("Training the tokenizer.")
    tokenizer.train_from_iterator(batch_iterator()  ,trainer=A_)
    logger.info("Tokenizer training complete!")

    UpperCamelCase__:					str												=      tokenizer.token_to_id("[CLS]")
    UpperCamelCase__:					str												=      tokenizer.token_to_id("[SEP]")
    UpperCamelCase__:					List[Any]												=      processors.TemplateProcessing(
        single="[CLS]:0 $A:0 [SEP]:0"  ,pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"  ,special_tokens=[
            ("[CLS]", cls_token_id),
            ("[SEP]", sep_token_id),
        ]  ,)
    UpperCamelCase__:					Tuple												=      decoders.Metaspace()

    if args.export_to_hub:
        logger.info("Exporting the trained tokenzier to Hub.")
        UpperCamelCase__:					List[Any]												=      AlbertTokenizerFast(tokenizer_object=A_)
        new_tokenizer.push_to_hub("unigram-tokenizer-dataset")


if __name__ == "__main__":
   A__:  Union[str, Any]       					=					parse_args()
   main(args)








def 							lowerCAmelCase_				(    A_  ,A_  ,A_):
    UpperCamelCase__:					Optional[int]												=      [1 / first_term]
    UpperCamelCase__:					int												=      1 / first_term
    for _ in range(number_of_terms - 1):
        first_term += common_difference
        arithmetic_progression.append(A_)
    UpperCamelCase__:					str												=      [1 / step for step in arithmetic_progression]
    return sum(A_)


if __name__ == "__main__":
   import doctest

   doctest.testmod()
   print(sum_of_harmonic_progression(1 / 2, 2, 2))

