'''simple docstring'''

import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_A      :				Optional[Any] 				=    logging.getLogger(__name__)







def 					UpperCamelCase_						( ) ->					List[str]:






 '''simple docstring'''

 __lowerCAmelCase    							=						argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""				)
 parser.add_argument(
     """--dataset_name"""						,		type=snake_case_						,		default="""wikitext"""						,		help="""Name of the training. Explore datasets at: hf.co/datasets."""						,		)
 parser.add_argument(
     """--dataset_config"""						,		type=snake_case_						,		default="""wikitext-103-raw-v1"""						,		help="""Configuration name of the dataset."""				)
 parser.add_argument(
     """--batch_size"""						,		type=snake_case_						,		default=10_00						,		help="""Batch size during training."""						,		)
 parser.add_argument(
     """--vocab_size"""						,		type=snake_case_						,		default=1_00_48						,		help="""Size of the desired vocabulary."""						,		)
 parser.add_argument(
     """--limit"""						,		default=snake_case_						,		type=snake_case_						,		help="""Limit the number of shards (used for debugging)."""						,		)
 parser.add_argument(
     """--export_to_hub"""						,		action="""store_true"""						,		)

 __lowerCAmelCase    							=						parser.parse_args()
 return args







def 					UpperCamelCase_						( snake_case_	:	Optional[Any]				) ->					List[Any]:






 '''simple docstring'''

 __lowerCAmelCase    							=						datasets.load_dataset(args.dataset_name						,		args.dataset_config						,		split="""train"""				)

 if args.limit is not None:
  __lowerCAmelCase    							=						min(len(snake_case_				)						,		args.limit				)
  __lowerCAmelCase    							=						dataset.select(range(snake_case_				)				)
  logger.info(f"""Limiting the dataset to {args.limit} entries."""				)

 def batch_iterator():
  for i in range(0						,		len(snake_case_				)						,		args.batch_size				):
   yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
 __lowerCAmelCase    							=						Tokenizer(Unigram()				)
 __lowerCAmelCase    							=						normalizers.Sequence([normalizers.Replace("""``"""						,		"""\""""				), normalizers.Replace("""''"""						,		"""\""""				)]				)
 __lowerCAmelCase    							=						pre_tokenizers.Metaspace()

 # Prepare the trainer.
 __lowerCAmelCase    							=						UnigramTrainer(
     unk_token="""<unk>"""						,		special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]						,		vocab_size=args.vocab_size						,		)

 logger.info("""Training the tokenizer."""				)
 tokenizer.train_from_iterator(batch_iterator()						,		trainer=snake_case_				)
 logger.info("""Tokenizer training complete!"""				)

 __lowerCAmelCase    							=						tokenizer.token_to_id("""[CLS]"""				)
 __lowerCAmelCase    							=						tokenizer.token_to_id("""[SEP]"""				)
 __lowerCAmelCase    							=						processors.TemplateProcessing(
     single="""[CLS]:0 $A:0 [SEP]:0"""						,		pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""						,		special_tokens=[
         ("""[CLS]""", cls_token_id),
         ("""[SEP]""", sep_token_id),
     ]						,		)
 __lowerCAmelCase    							=						decoders.Metaspace()

 if args.export_to_hub:
  logger.info("""Exporting the trained tokenzier to Hub."""				)
  __lowerCAmelCase    							=						AlbertTokenizerFast(tokenizer_object=snake_case_				)
  new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""				)


if __name__ == "__main__":
 _A      :				Union[str, Any] 				=    parse_args()
 main(args)







'''simple docstring'''

def 					UpperCamelCase_						( snake_case_	:	int				) ->					int:






 '''simple docstring'''

 if number != int(snake_case_				):
  raise ValueError("""factorial() only accepts integral values"""				)
 if number < 0:
  raise ValueError("""factorial() not defined for negative values"""				)
 __lowerCAmelCase    							=						1
 for i in range(1						,		number + 1				):
  value *= i
 return value







def 					UpperCamelCase_						( snake_case_	:	int				) ->					int:






 '''simple docstring'''

 if not isinstance(snake_case_						,		snake_case_				):
  raise ValueError("""factorial() only accepts integral values"""				)
 if n < 0:
  raise ValueError("""factorial() not defined for negative values"""				)
 return 1 if n == 0 or n == 1 else n * factorial(n - 1				)


if __name__ == "__main__":
 import doctest

 doctest.testmod()

 _A      :				List[Any] 				=    int(input('''Enter a positive integer: ''').strip() or 0)
 print(f'factorial{n} is {factorial(n)}')






