



import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


A__     				=	logging.getLogger(__name__)


def 		_UpperCAmelCase    (				):





     """simple docstring"""

     _lowerCAmelCase      =   argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""      )
     parser.add_argument(
         """--dataset_name"""     ,						type=snake_case     ,						default="""wikitext"""     ,						help="""Name of the training. Explore datasets at: hf.co/datasets."""     ,						)
     parser.add_argument(
         """--dataset_config"""     ,						type=snake_case     ,						default="""wikitext-103-raw-v1"""     ,						help="""Configuration name of the dataset."""      )
     parser.add_argument(
         """--batch_size"""     ,						type=snake_case     ,						default=10_00     ,						help="""Batch size during training."""     ,						)
     parser.add_argument(
         """--vocab_size"""     ,						type=snake_case     ,						default=1_00_48     ,						help="""Size of the desired vocabulary."""     ,						)
     parser.add_argument(
         """--limit"""     ,						default=snake_case     ,						type=snake_case     ,						help="""Limit the number of shards (used for debugging)."""     ,						)
     parser.add_argument(
         """--export_to_hub"""     ,						action="""store_true"""     ,						)

     _lowerCAmelCase      =   parser.parse_args()
     return args


def 		_UpperCAmelCase    (				snake_case      ):





     """simple docstring"""

     _lowerCAmelCase      =   datasets.load_dataset(args.dataset_name     ,						args.dataset_config     ,						split="""train"""      )

     if args.limit is not None:
          _lowerCAmelCase      =   min(len(snake_case      )     ,						args.limit      )
          _lowerCAmelCase      =   dataset.select(range(snake_case      )      )
          logger.info(F'Limiting the dataset to {args.limit} entries.'      )

     def batch_iterator():
          for i in range(0     ,						len(snake_case      )     ,						args.batch_size      ):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     _lowerCAmelCase      =   Tokenizer(Unigram()      )
     _lowerCAmelCase      =   normalizers.Sequence([normalizers.Replace("""``"""     ,						"""\""""      ), normalizers.Replace("""''"""     ,						"""\""""      )]      )
     _lowerCAmelCase      =   pre_tokenizers.Metaspace()

     # Prepare the trainer.
     _lowerCAmelCase      =   UnigramTrainer(
         unk_token="""<unk>"""     ,						special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]     ,						vocab_size=args.vocab_size     ,						)

     logger.info("""Training the tokenizer."""      )
     tokenizer.train_from_iterator(batch_iterator()     ,						trainer=snake_case      )
     logger.info("""Tokenizer training complete!"""      )

     _lowerCAmelCase      =   tokenizer.token_to_id("""[CLS]"""      )
     _lowerCAmelCase      =   tokenizer.token_to_id("""[SEP]"""      )
     _lowerCAmelCase      =   processors.TemplateProcessing(
         single="""[CLS]:0 $A:0 [SEP]:0"""     ,						pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""     ,						special_tokens=[
             ("""[CLS]""", cls_token_id),
             ("""[SEP]""", sep_token_id),
         ]     ,						)
     _lowerCAmelCase      =   decoders.Metaspace()

     if args.export_to_hub:
          logger.info("""Exporting the trained tokenzier to Hub."""      )
          _lowerCAmelCase      =   AlbertTokenizerFast(tokenizer_object=snake_case      )
          new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""      )


if __name__ == "__main__":
 A__     				=	parse_args()
 main(args)








from math import exp


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					return 8 * x - 2 * exp(-x      )


def 		_UpperCAmelCase    (				snake_case     ,						snake_case     ,						snake_case      ):





					"""simple docstring"""

					_lowerCAmelCase      =   lower_bound
					_lowerCAmelCase      =   upper_bound
					for _ in range(0     ,						snake_case      ):
										_lowerCAmelCase     ,     _lowerCAmelCase      =   xa, xa - (f(snake_case      ) * (xa - xa)) / (f(snake_case      ) - f(snake_case      ))
					return xa


if __name__ == "__main__":
	print(f"Example: {secant_method(1, 3, 2)}")



