


'''simple docstring'''



import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


A__		: Optional[int]												=  logging.getLogger(__name__)



def       UpperCAmelCase__ (     )					->		Dict:
     __lowerCamelCase       :     Tuple        =		argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'							)
     parser.add_argument(
         '--dataset_name'				,   type=UpperCAmelCase_				,   default='wikitext'				,   help='Name of the training. Explore datasets at: hf.co/datasets.'				,   )
     parser.add_argument(
         '--dataset_config'				,   type=UpperCAmelCase_				,   default='wikitext-103-raw-v1'				,   help='Configuration name of the dataset.'							)
     parser.add_argument(
         '--batch_size'				,   type=UpperCAmelCase_				,   default=10_00				,   help='Batch size during training.'				,   )
     parser.add_argument(
         '--vocab_size'				,   type=UpperCAmelCase_				,   default=1_00_48				,   help='Size of the desired vocabulary.'				,   )
     parser.add_argument(
         '--limit'				,   default=UpperCAmelCase_				,   type=UpperCAmelCase_				,   help='Limit the number of shards (used for debugging).'				,   )
     parser.add_argument(
         '--export_to_hub'				,   action='store_true'				,   )

     __lowerCamelCase       :     Tuple        =		parser.parse_args()
     return args



def       UpperCAmelCase__ (     UpperCAmelCase_   :    Optional[int]							)					->		Dict:
     __lowerCamelCase       :     Union[str, Any]        =		datasets.load_dataset(args.dataset_name				,   args.dataset_config				,   split='train'							)

     if args.limit is not None:
          __lowerCamelCase       :     List[str]        =		min(len(UpperCAmelCase_							)				,   args.limit							)
          __lowerCamelCase       :     Optional[Any]        =		dataset.select(range(UpperCAmelCase_							)							)
          logger.info(F'Limiting the dataset to {args.limit} entries.'							)

     def batch_iterator():
          for i in range(0				,   len(UpperCAmelCase_							)				,   args.batch_size							):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     __lowerCamelCase       :     Optional[int]        =		Tokenizer(Unigram()							)
     __lowerCamelCase       :     Any        =		normalizers.Sequence([normalizers.Replace('``'				,   '"'							), normalizers.Replace('\'\''				,   '"'							)]							)
     __lowerCamelCase       :     List[Any]        =		pre_tokenizers.Metaspace()

     # Prepare the trainer.
     __lowerCamelCase       :     List[str]        =		UnigramTrainer(
         unk_token='<unk>'				,   special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']				,   vocab_size=args.vocab_size				,   )

     logger.info('Training the tokenizer.'							)
     tokenizer.train_from_iterator(batch_iterator()				,   trainer=UpperCAmelCase_							)
     logger.info('Tokenizer training complete!'							)

     __lowerCamelCase       :     Optional[Any]        =		tokenizer.token_to_id('[CLS]'							)
     __lowerCamelCase       :     List[Any]        =		tokenizer.token_to_id('[SEP]'							)
     __lowerCamelCase       :     Union[str, Any]        =		processors.TemplateProcessing(
         single='[CLS]:0 $A:0 [SEP]:0'				,   pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'				,   special_tokens=[
             ('[CLS]', cls_token_id),
             ('[SEP]', sep_token_id),
         ]				,   )
     __lowerCamelCase       :     List[str]        =		decoders.Metaspace()

     if args.export_to_hub:
          logger.info('Exporting the trained tokenzier to Hub.'							)
          __lowerCamelCase       :     List[str]        =		AlbertTokenizerFast(tokenizer_object=UpperCAmelCase_							)
          new_tokenizer.push_to_hub('unigram-tokenizer-dataset'							)


if __name__ == "__main__":
      A__		: Union[str, Any]												=  parse_args()
      main(args)










'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


A__		: str												=  {"""configuration_hubert""": ["""HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """HubertConfig"""]}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__		: List[str]												=  [
          """HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """HubertForCTC""",
          """HubertForSequenceClassification""",
          """HubertModel""",
          """HubertPreTrainedModel""",
      ]


try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__		: Optional[int]												=  [
          """TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """TFHubertForCTC""",
          """TFHubertModel""",
          """TFHubertPreTrainedModel""",
      ]

if TYPE_CHECKING:
      from .configuration_hubert import HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, HubertConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_hubert import (
                HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                HubertForCTC,
                HubertForSequenceClassification,
                HubertModel,
                HubertPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_hubert import (
                TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFHubertForCTC,
                TFHubertModel,
                TFHubertPreTrainedModel,
            )


else:
      import sys

      A__		: Optional[int]												=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






