


'''simple docstring'''


from arguments import TokenizerTrainingArguments
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, HfArgumentParser
from transformers.models.gpta.tokenization_gpta import bytes_to_unicode







def       snake_case_       (  SCREAMING_SNAKE_CASE__=10					):







				"""simple docstring"""





				for _ in tqdm(range(0							,				args.n_examples							,				SCREAMING_SNAKE_CASE__					)					):
								yield [next(SCREAMING_SNAKE_CASE__					)[args.text_column] for _ in range(SCREAMING_SNAKE_CASE__					)]


# Configuration
UpperCAmelCase_		:       List[Any]							       =		HfArgumentParser(TokenizerTrainingArguments)
UpperCAmelCase_		:       Any							       =		parser.parse_args()

# Base tokenizer
UpperCAmelCase_		:       List[Any]							       =		AutoTokenizer.from_pretrained(args.base_tokenizer)
UpperCAmelCase_		:       Optional[int]							       =		list(bytes_to_unicode().values())

# Load dataset
UpperCAmelCase_		:       Optional[Any]							       =		load_dataset(args.dataset_name, split='train', streaming=True)
UpperCAmelCase_		:       Union[str, Any]							       =		iter(dataset)


# Training and saving
UpperCAmelCase_		:       List[str]							       =		tokenizer.train_new_from_iterator(
    batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
)







new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)








'''simple docstring'''


def       snake_case_       (  SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	Tuple							=				1
				while b > 0:
								if b & 1:
												res *= a

								a *= a
								b >>= 1

				return res







def       snake_case_       (  SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__							,				SCREAMING_SNAKE_CASE__					):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE			:	Union[str, Any]							=				1
				while b > 0:
								if b & 1:
												_SCREAMING_SNAKE_CASE			:	Tuple							=				((res % c) * (a % c)) % c

								a *= a
								b >>= 1

				return res




