






'''simple docstring'''





import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_UpperCamelCase		:		int								=			logging.getLogger(__name__)
def __UpperCAmelCase (			)  -> int:
   UpperCAmelCase_     :   Union[str, Any]       =   argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''						)
   parser.add_argument(
       '''--dataset_name'''	,   type=A_	,   default='''wikitext'''	,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''	,   )
   parser.add_argument(
       '''--dataset_config'''	,   type=A_	,   default='''wikitext-103-raw-v1'''	,   help='''Configuration name of the dataset.'''						)
   parser.add_argument(
       '''--batch_size'''	,   type=A_	,   default=1_0_0_0	,   help='''Batch size during training.'''	,   )
   parser.add_argument(
       '''--vocab_size'''	,   type=A_	,   default=1_0_0_4_8	,   help='''Size of the desired vocabulary.'''	,   )
   parser.add_argument(
       '''--limit'''	,   default=A_	,   type=A_	,   help='''Limit the number of shards (used for debugging).'''	,   )
   parser.add_argument(
       '''--export_to_hub'''	,   action='''store_true'''	,   )

   UpperCAmelCase_     :   Tuple       =   parser.parse_args()
   return args
def __UpperCAmelCase (			A			:	List[Any]						)  -> str:
   UpperCAmelCase_     :   Dict       =   datasets.load_dataset(args.dataset_name	,   args.dataset_config	,   split='''train'''						)

   if args.limit is not None:
      UpperCAmelCase_     :   Optional[Any]       =   min(len(A_						)	,   args.limit						)
      UpperCAmelCase_     :   List[Any]       =   dataset.select(range(A_						)						)
      logger.info(F"Limiting the dataset to {args.limit} entries."						)

   def batch_iterator():
      for i in range(0	,   len(A_						)	,   args.batch_size						):
         yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
   UpperCAmelCase_     :   Union[str, Any]       =   Tokenizer(Unigram()						)
   UpperCAmelCase_     :   List[str]       =   normalizers.Sequence([normalizers.Replace('''``'''	,   '''\"'''						), normalizers.Replace('''\'\''''	,   '''\"'''						)]						)
   UpperCAmelCase_     :   Optional[Any]       =   pre_tokenizers.Metaspace()

   # Prepare the trainer.
   UpperCAmelCase_     :   List[str]       =   UnigramTrainer(
       unk_token='''<unk>'''	,   special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]''']	,   vocab_size=args.vocab_size	,   )

   logger.info('''Training the tokenizer.'''						)
   tokenizer.train_from_iterator(batch_iterator()	,   trainer=A_						)
   logger.info('''Tokenizer training complete!'''						)

   UpperCAmelCase_     :   str       =   tokenizer.token_to_id('''[CLS]'''						)
   UpperCAmelCase_     :   str       =   tokenizer.token_to_id('''[SEP]'''						)
   UpperCAmelCase_     :   List[Any]       =   processors.TemplateProcessing(
       single='''[CLS]:0 $A:0 [SEP]:0'''	,   pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'''	,   special_tokens=[
           ('''[CLS]''', cls_token_id),
           ('''[SEP]''', sep_token_id),
       ]	,   )
   UpperCAmelCase_     :   Tuple       =   decoders.Metaspace()

   if args.export_to_hub:
      logger.info('''Exporting the trained tokenzier to Hub.'''						)
      UpperCAmelCase_     :   List[Any]       =   AlbertTokenizerFast(tokenizer_object=A_						)
      new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''						)


if __name__ == "__main__":
      _UpperCamelCase		:		Union[str, Any]								=			parse_args()
      main(args)










'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available


_UpperCamelCase		:		str								=			{}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_UpperCamelCase		:		List[Any]								=			['MBart50Tokenizer']

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_UpperCamelCase		:		List[str]								=			['MBart50TokenizerFast']


if TYPE_CHECKING:
						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_mbartaa import MBartaaTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_mbartaa_fast import MBartaaTokenizerFast

else:
						import sys

						_UpperCamelCase		:		Optional[int]								=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


