



import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


A__     				=	logging.getLogger(__name__)


def 		_UpperCAmelCase    (				):





					"""simple docstring"""

					_lowerCAmelCase      =   argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""      )
					parser.add_argument(
					    """--dataset_name"""     ,						type=snake_case     ,						default="""wikitext"""     ,						help="""Name of the training. Explore datasets at: hf.co/datasets."""     ,						)
					parser.add_argument(
					    """--dataset_config"""     ,						type=snake_case     ,						default="""wikitext-103-raw-v1"""     ,						help="""Configuration name of the dataset."""      )
					parser.add_argument(
					    """--batch_size"""     ,						type=snake_case     ,						default=10_00     ,						help="""Batch size during training."""     ,						)
					parser.add_argument(
					    """--vocab_size"""     ,						type=snake_case     ,						default=1_00_48     ,						help="""Size of the desired vocabulary."""     ,						)
					parser.add_argument(
					    """--limit"""     ,						default=snake_case     ,						type=snake_case     ,						help="""Limit the number of shards (used for debugging)."""     ,						)
					parser.add_argument(
					    """--export_to_hub"""     ,						action="""store_true"""     ,						)

					_lowerCAmelCase      =   parser.parse_args()
					return args


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					_lowerCAmelCase      =   datasets.load_dataset(args.dataset_name     ,						args.dataset_config     ,						split="""train"""      )

					if args.limit is not None:
										_lowerCAmelCase      =   min(len(snake_case      )     ,						args.limit      )
										_lowerCAmelCase      =   dataset.select(range(snake_case      )      )
										logger.info(F'Limiting the dataset to {args.limit} entries.'      )

					def batch_iterator():
										for i in range(0     ,						len(snake_case      )     ,						args.batch_size      ):
															yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
					_lowerCAmelCase      =   Tokenizer(Unigram()      )
					_lowerCAmelCase      =   normalizers.Sequence([normalizers.Replace("""``"""     ,						"""\""""      ), normalizers.Replace("""''"""     ,						"""\""""      )]      )
					_lowerCAmelCase      =   pre_tokenizers.Metaspace()

					# Prepare the trainer.
					_lowerCAmelCase      =   UnigramTrainer(
					    unk_token="""<unk>"""     ,						special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]     ,						vocab_size=args.vocab_size     ,						)

					logger.info("""Training the tokenizer."""      )
					tokenizer.train_from_iterator(batch_iterator()     ,						trainer=snake_case      )
					logger.info("""Tokenizer training complete!"""      )

					_lowerCAmelCase      =   tokenizer.token_to_id("""[CLS]"""      )
					_lowerCAmelCase      =   tokenizer.token_to_id("""[SEP]"""      )
					_lowerCAmelCase      =   processors.TemplateProcessing(
					    single="""[CLS]:0 $A:0 [SEP]:0"""     ,						pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""     ,						special_tokens=[
					        ("""[CLS]""", cls_token_id),
					        ("""[SEP]""", sep_token_id),
					    ]     ,						)
					_lowerCAmelCase      =   decoders.Metaspace()

					if args.export_to_hub:
										logger.info("""Exporting the trained tokenzier to Hub."""      )
										_lowerCAmelCase      =   AlbertTokenizerFast(tokenizer_object=snake_case      )
										new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""      )


if __name__ == "__main__":
	A__     				=	parse_args()
	main(args)








# Lint as: python3

import hashlib
from typing import Union


def 		_UpperCAmelCase    (				snake_case      ):





					"""simple docstring"""

					if isinstance(snake_case     ,						snake_case      ):
										# Data already in bytes, returns as it as
										return hash_data
					elif isinstance(snake_case     ,						snake_case      ):
										# We keep the data as it as for it ot be later encoded to UTF-8
										# However replace `\\` with `/` for Windows compatibility
										_lowerCAmelCase      =   hash_data.replace("""\\"""     ,						"""/"""      )
					elif isinstance(snake_case     ,						snake_case      ):
										_lowerCAmelCase      =   str(snake_case      )
					else:
										# If data is not of the required type, raise error
										raise InvalidKeyError(snake_case      )

					return hash_data.encode("""utf-8"""      )





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   """\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
									_lowerCAmelCase      =   F'\nFound Key {hash_data} of type {type(_snake_case		)}'
									_lowerCAmelCase      =   """\nKeys should be either str, int or bytes type"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			(					lowerCamelCase__	):




				def __init__(							self						,    _snake_case						,    _snake_case						,    _snake_case=""		):



									"""simple docstring"""





									_lowerCAmelCase      =   key
									_lowerCAmelCase      =   duplicate_key_indices
									_lowerCAmelCase      =   fix_msg
									_lowerCAmelCase      =   """Found multiple examples generated with the same key"""
									if len(_snake_case		) <= 20:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(_snake_case		)} have the key {key}'
									else:
														_lowerCAmelCase      =   F'\nThe examples at index {", ".join(duplicate_key_indices[:20]		)}... ({len(_snake_case		) - 20} more) have the key {key}'
									_lowerCAmelCase      =   """\n""" + fix_msg if fix_msg else """"""
									super().__init__(F'{self.prefix}{self.err_msg}{self.suffix}'		)





class  __lowerCAmelCase			:




				def __init__(							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   hashlib.mda(_as_bytes(_snake_case		)		)




				def 							snake_case       (							self						,    _snake_case		):



									"""simple docstring"""





									_lowerCAmelCase      =   self._split_mda.copy()
									_lowerCAmelCase      =   _as_bytes(_snake_case		)
									mda.update(_snake_case		)
									# Convert to integer with hexadecimal conversion
									return int(mda.hexdigest()						,    16		)



