


'''simple docstring'''


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


UpperCAmelCase_		:       Tuple							       =		logging.getLogger(__name__)







def       snake_case_       (  ):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	Optional[Any]							=				argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""					)
				parser.add_argument(
				    """--dataset_name"""							,				type=SCREAMING_SNAKE_CASE__							,				default="""wikitext"""							,				help="""Name of the training. Explore datasets at: hf.co/datasets."""							,				)
				parser.add_argument(
				    """--dataset_config"""							,				type=SCREAMING_SNAKE_CASE__							,				default="""wikitext-103-raw-v1"""							,				help="""Configuration name of the dataset."""					)
				parser.add_argument(
				    """--batch_size"""							,				type=SCREAMING_SNAKE_CASE__							,				default=1000							,				help="""Batch size during training."""							,				)
				parser.add_argument(
				    """--vocab_size"""							,				type=SCREAMING_SNAKE_CASE__							,				default=1_0048							,				help="""Size of the desired vocabulary."""							,				)
				parser.add_argument(
				    """--limit"""							,				default=SCREAMING_SNAKE_CASE__							,				type=SCREAMING_SNAKE_CASE__							,				help="""Limit the number of shards (used for debugging)."""							,				)
				parser.add_argument(
				    """--export_to_hub"""							,				action="""store_true"""							,				)

				_SCREAMING_SNAKE_CASE			:	Any							=				parser.parse_args()
				return args







def       snake_case_       (  SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	List[str]							=				datasets.load_dataset(args.dataset_name							,				args.dataset_config							,				split="""train"""					)

				if args.limit is not None:
								_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				min(len(SCREAMING_SNAKE_CASE__					)							,				args.limit					)
								_SCREAMING_SNAKE_CASE			:	Optional[int]							=				dataset.select(range(SCREAMING_SNAKE_CASE__					)					)
								logger.info(f"""Limiting the dataset to {args.limit} entries."""					)

				def batch_iterator():
								for i in range(0							,				len(SCREAMING_SNAKE_CASE__					)							,				args.batch_size					):
												yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
				_SCREAMING_SNAKE_CASE			:	int							=				Tokenizer(Unigram()					)
				_SCREAMING_SNAKE_CASE			:	List[str]							=				normalizers.Sequence([normalizers.Replace("""``"""							,				"""\""""					), normalizers.Replace("""''"""							,				"""\""""					)]					)
				_SCREAMING_SNAKE_CASE			:	Optional[int]							=				pre_tokenizers.Metaspace()

				# Prepare the trainer.
				_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				UnigramTrainer(
				    unk_token="""<unk>"""							,				special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]							,				vocab_size=args.vocab_size							,				)

				logger.info("""Training the tokenizer."""					)
				tokenizer.train_from_iterator(batch_iterator()							,				trainer=SCREAMING_SNAKE_CASE__					)
				logger.info("""Tokenizer training complete!"""					)

				_SCREAMING_SNAKE_CASE			:	List[Any]							=				tokenizer.token_to_id("""[CLS]"""					)
				_SCREAMING_SNAKE_CASE			:	int							=				tokenizer.token_to_id("""[SEP]"""					)
				_SCREAMING_SNAKE_CASE			:	Optional[int]							=				processors.TemplateProcessing(
				    single="""[CLS]:0 $A:0 [SEP]:0"""							,				pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""							,				special_tokens=[
				        ("""[CLS]""", cls_token_id),
				        ("""[SEP]""", sep_token_id),
				    ]							,				)
				_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				decoders.Metaspace()

				if args.export_to_hub:
								logger.info("""Exporting the trained tokenzier to Hub."""					)
								_SCREAMING_SNAKE_CASE			:	Tuple							=				AlbertTokenizerFast(tokenizer_object=SCREAMING_SNAKE_CASE__					)
								new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""					)


if __name__ == "__main__":
				UpperCAmelCase_		:       Union[str, Any]							       =		parse_args()
				main(args)








'''simple docstring'''


import sys







def       snake_case_       (  SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				if not numbers:
								return 0
				if target == 0 and target in numbers:
								return 0
				if not isinstance(SCREAMING_SNAKE_CASE__							,				(list, tuple)					) or not all(
				    isinstance(SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__					) for number in numbers					):
								raise ValueError("""numbers must be an iterable of integers"""					)

				_SCREAMING_SNAKE_CASE			:	str							=				0
				_SCREAMING_SNAKE_CASE			:	int							=				sys.maxsize

				while right < len(SCREAMING_SNAKE_CASE__					):
								curr_sum += numbers[right]
								while curr_sum >= target and left <= right:
												_SCREAMING_SNAKE_CASE			:	Any							=				min(SCREAMING_SNAKE_CASE__							,				right - left + 1					)
												curr_sum -= numbers[left]
												left += 1
								right += 1

				return 0 if min_len == sys.maxsize else min_len




