





'''simple docstring'''




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


__UpperCAmelCase         =   logging.getLogger(__name__)



def 						__A						(       ):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE				:							Optional[Any]						=	argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""					)
		parser.add_argument(
		    """--dataset_name"""	,	type=lowerCamelCase_	,	default="""wikitext"""	,	help="""Name of the training. Explore datasets at: hf.co/datasets."""	,	)
		parser.add_argument(
		    """--dataset_config"""	,	type=lowerCamelCase_	,	default="""wikitext-103-raw-v1"""	,	help="""Configuration name of the dataset."""					)
		parser.add_argument(
		    """--batch_size"""	,	type=lowerCamelCase_	,	default=10_00	,	help="""Batch size during training."""	,	)
		parser.add_argument(
		    """--vocab_size"""	,	type=lowerCamelCase_	,	default=1_00_48	,	help="""Size of the desired vocabulary."""	,	)
		parser.add_argument(
		    """--limit"""	,	default=lowerCamelCase_	,	type=lowerCamelCase_	,	help="""Limit the number of shards (used for debugging)."""	,	)
		parser.add_argument(
		    """--export_to_hub"""	,	action="""store_true"""	,	)

		SCREAMING_SNAKE_CASE				:							List[str]						=	parser.parse_args()
		return args



def 						__A						(       lowerCamelCase_					):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE				:							Optional[Any]						=	datasets.load_dataset(args.dataset_name	,	args.dataset_config	,	split="""train"""					)

		if args.limit is not None:
				SCREAMING_SNAKE_CASE				:							Optional[Any]						=	min(len(lowerCamelCase_					)	,	args.limit					)
				SCREAMING_SNAKE_CASE				:							Optional[Any]						=	dataset.select(range(lowerCamelCase_					)					)
				logger.info(f'''Limiting the dataset to {args.limit} entries.'''					)

		def batch_iterator():
				for i in range(0	,	len(lowerCamelCase_					)	,	args.batch_size					):
						yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
		SCREAMING_SNAKE_CASE				:							str						=	Tokenizer(Unigram()					)
		SCREAMING_SNAKE_CASE				:							Optional[int]						=	normalizers.Sequence([normalizers.Replace("""``"""	,	"""\""""					), normalizers.Replace("""''"""	,	"""\""""					)]					)
		SCREAMING_SNAKE_CASE				:							List[str]						=	pre_tokenizers.Metaspace()

		# Prepare the trainer.
		SCREAMING_SNAKE_CASE				:							Dict						=	UnigramTrainer(
		    unk_token="""<unk>"""	,	special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]	,	vocab_size=args.vocab_size	,	)

		logger.info("""Training the tokenizer."""					)
		tokenizer.train_from_iterator(batch_iterator()	,	trainer=lowerCamelCase_					)
		logger.info("""Tokenizer training complete!"""					)

		SCREAMING_SNAKE_CASE				:							Optional[Any]						=	tokenizer.token_to_id("""[CLS]"""					)
		SCREAMING_SNAKE_CASE				:							str						=	tokenizer.token_to_id("""[SEP]"""					)
		SCREAMING_SNAKE_CASE				:							Tuple						=	processors.TemplateProcessing(
		    single="""[CLS]:0 $A:0 [SEP]:0"""	,	pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""	,	special_tokens=[
		        ("""[CLS]""", cls_token_id),
		        ("""[SEP]""", sep_token_id),
		    ]	,	)
		SCREAMING_SNAKE_CASE				:							str						=	decoders.Metaspace()

		if args.export_to_hub:
				logger.info("""Exporting the trained tokenzier to Hub."""					)
				SCREAMING_SNAKE_CASE				:							List[Any]						=	AlbertTokenizerFast(tokenizer_object=lowerCamelCase_					)
				new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""					)


if __name__ == "__main__":
							__UpperCAmelCase         =   parse_args()
							main(args)










'''simple docstring'''




from __future__ import annotations

from collections import Counter



def 						__A						(       lowerCamelCase_					):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE				:							int						=	[True] * n
		SCREAMING_SNAKE_CASE				:							Optional[Any]						=	False
		SCREAMING_SNAKE_CASE				:							Optional[Any]						=	False
		SCREAMING_SNAKE_CASE				:							Optional[int]						=	True

		for i in range(3	,	int(n**0.5 + 1					)	,	2					):
				SCREAMING_SNAKE_CASE				:							List[Any]						=	i * 2
				while index < n:
						SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	False
						SCREAMING_SNAKE_CASE				:							Optional[Any]						=	index + i

		SCREAMING_SNAKE_CASE				:							Dict						=	[2]

		for i in range(3	,	lowerCamelCase_	,	2					):
				if is_prime[i]:
						primes.append(lowerCamelCase_					)

		return primes



def 						__A						(       lowerCamelCase_					):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE				:							List[str]						=	str(lowerCamelCase_					)
		SCREAMING_SNAKE_CASE				:							List[str]						=	[]
		SCREAMING_SNAKE_CASE				:							Optional[int]						=	["""0""", """1""", """2""", """3""", """4""", """5""", """6""", """7""", """8""", """9"""]

		for duplicate in Counter(lowerCamelCase_					) - Counter(set(lowerCamelCase_					)					):
				SCREAMING_SNAKE_CASE				:							Tuple						=	[int(number_str.replace(lowerCamelCase_	,	lowerCamelCase_					)					) for digit in digits]
				replacements.append(lowerCamelCase_					)

		return replacements



def 						__A						(       lowerCamelCase_ = 8					):


		"""simple docstring"""



		SCREAMING_SNAKE_CASE				:							Tuple						=	set()

		# Filter primes with less than 3 replaceable digits
		SCREAMING_SNAKE_CASE				:							Union[str, Any]						=	{
		    x for x in set(prime_sieve(1_00_00_00					)					) if len(str(lowerCamelCase_					)					) - len(set(str(lowerCamelCase_					)					)					) >= 3
		}

		for prime in primes:
				if prime in numbers_checked:
						continue

				SCREAMING_SNAKE_CASE				:							int						=	digit_replacements(lowerCamelCase_					)

				for family in replacements:
						numbers_checked.update(lowerCamelCase_					)
						SCREAMING_SNAKE_CASE				:							Dict						=	primes.intersection(lowerCamelCase_					)

						if len(lowerCamelCase_					) != family_length:
								continue

						return min(lowerCamelCase_					)

		return -1


if __name__ == "__main__":
							print(solution())



