





"""simple docstring"""




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_a									=				logging.getLogger(__name__)





def lowerCAmelCase__()    ->				Tuple:







	'''simple docstring'''







	lowerCamelCase__        =		argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''							)
	parser.add_argument(
	    '''--dataset_name'''	,type=A__	,default='''wikitext'''	,help='''Name of the training. Explore datasets at: hf.co/datasets.'''	,)
	parser.add_argument(
	    '''--dataset_config'''	,type=A__	,default='''wikitext-103-raw-v1'''	,help='''Configuration name of the dataset.'''							)
	parser.add_argument(
	    '''--batch_size'''	,type=A__	,default=1000	,help='''Batch size during training.'''	,)
	parser.add_argument(
	    '''--vocab_size'''	,type=A__	,default=10048	,help='''Size of the desired vocabulary.'''	,)
	parser.add_argument(
	    '''--limit'''	,default=A__	,type=A__	,help='''Limit the number of shards (used for debugging).'''	,)
	parser.add_argument(
	    '''--export_to_hub'''	,action='''store_true'''	,)

	lowerCamelCase__        =		parser.parse_args()
	return args





def lowerCAmelCase__(__snake_case							)    ->				List[str]:







	'''simple docstring'''







	lowerCamelCase__        =		datasets.load_dataset(args.dataset_name	,args.dataset_config	,split='''train'''							)

	if args.limit is not None:
		lowerCamelCase__        =		min(len(A__							)	,args.limit							)
		lowerCamelCase__        =		dataset.select(range(A__							)							)
		logger.info(F'Limiting the dataset to {args.limit} entries.'							)

	def batch_iterator():
		for i in range(0	,len(A__							)	,args.batch_size							):
			yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
	lowerCamelCase__        =		Tokenizer(Unigram()							)
	lowerCamelCase__        =		normalizers.Sequence([normalizers.Replace('''``'''	,'''"'''							), normalizers.Replace('''\'\''''	,'''"'''							)]							)
	lowerCamelCase__        =		pre_tokenizers.Metaspace()

	# Prepare the trainer.
	lowerCamelCase__        =		UnigramTrainer(
	    unk_token='''<unk>'''	,special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]''']	,vocab_size=args.vocab_size	,)

	logger.info('''Training the tokenizer.'''							)
	tokenizer.train_from_iterator(batch_iterator()	,trainer=A__							)
	logger.info('''Tokenizer training complete!'''							)

	lowerCamelCase__        =		tokenizer.token_to_id('''[CLS]'''							)
	lowerCamelCase__        =		tokenizer.token_to_id('''[SEP]'''							)
	lowerCamelCase__        =		processors.TemplateProcessing(
	    single='''[CLS]:0 $A:0 [SEP]:0'''	,pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'''	,special_tokens=[
	        ('''[CLS]''', cls_token_id),
	        ('''[SEP]''', sep_token_id),
	    ]	,)
	lowerCamelCase__        =		decoders.Metaspace()

	if args.export_to_hub:
		logger.info('''Exporting the trained tokenzier to Hub.'''							)
		lowerCamelCase__        =		AlbertTokenizerFast(tokenizer_object=A__							)
		new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''							)


if __name__ == "__main__":
			_a									=				parse_args()
			main(args)










import os





def lowerCAmelCase__()    ->				Optional[Any]:







	'''simple docstring'''







	lowerCamelCase__        =		os.path.dirname(os.path.realpath(__snake_case							)							)
	lowerCamelCase__        =		os.path.join(__snake_case	,'''triangle.txt'''							)

	with open(__snake_case							) as f:
		lowerCamelCase__        =		f.readlines()

	lowerCamelCase__        =		[[int(__snake_case							) for y in x.rstrip('''\r\n'''							).split(''' '''							)] for x in triangle]

	for i in range(1	,len(__snake_case							)							):
		for j in range(len(a[i]							)							):
			lowerCamelCase__        =		a[i - 1][j] if j != len(a[i - 1]							) else 0
			lowerCamelCase__        =		a[i - 1][j - 1] if j > 0 else 0
			a[i][j] += max(__snake_case	,__snake_case							)
	return max(a[-1]							)


if __name__ == "__main__":
			print(solution())



