






from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

A__     :      Optional[Any]             =			{
    name: (getattr(transformers, name), getattr(transformers, name + 'Fast')) for name in SLOW_TO_FAST_CONVERTERS
}

A__     :      List[str]             =			datasets.load_dataset('xnli', split='test+validation')

A__     :      Union[str, Any]             =			0
A__     :      Dict             =			0
A__     :      str             =			0
A__     :      Tuple             =			0






def   a      (						lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				if spm_diff == list(reversed(lowerCamelCase_    )    ):
								# AAA -> AA+A vs A+AA case.
								return True
				elif len(lowerCamelCase_    ) == len(lowerCamelCase_    ) and fast.decode(lowerCamelCase_    ) == fast.decode(lowerCamelCase_    ):
								# Second order OK
								# Barrich -> Barr + ich vs Bar + rich
								return True
				lowercase__       			=      slow.encode(slow.decode(lowerCamelCase_    )    )
				lowercase__       			=      fast.encode(fast.decode(lowerCamelCase_    )    )
				if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
								# Type 3 error.
								# Snehagatha ->
								#       Sne, h, aga, th, a
								#       Sne, ha, gat, ha
								# Encoding the wrong with sp does not even recover what spm gave us
								# It fits tokenizer however...
								return True
				return False






def   a      (						lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				lowercase__       			=      fast.encode_plus(lowerCamelCase_    )[0]
				lowercase__       			=      enc.offsets
				lowercase__			,						lowercase__       			=      offsets[idx], offsets[idx - 1]
				if curr is not None and line[curr[0] : curr[1]] == "\u200f":
								return True
				if prev is not None and line[prev[0] : prev[1]] == "\u200f":
								return True






def   a      (						lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				# Encoding can be the same with same result AAA -> A + AA vs AA + A
				# We can check that we use at least exactly the same number of tokens.
				for i, (spm_id, tok_id) in enumerate(zip(lowerCamelCase_		,       lowerCamelCase_    )    ):
								if spm_id != tok_id:
												break
				lowercase__       			=      i
				for i, (spm_id, tok_id) in enumerate(zip(reversed(lowerCamelCase_    )		,       reversed(lowerCamelCase_    )    )    ):
								if spm_id != tok_id:
												break
				lowercase__       			=      len(lowerCamelCase_    ) - i

				lowercase__       			=      spm_ids[first:last]
				lowercase__       			=      tok_ids[first:last]

				if check_diff(lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):
								return True

				if check_LTR_mark(lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):
								return True

				if last - first > 5:
								# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
								lowercase__       			=      Counter(spm_ids[first:last]    )
								lowercase__       			=      Counter(tok_ids[first:last]    )

								lowercase__       			=      {spm_ for (spm_, si) in spms.items() if toks.get(spm_		,       0    ) == si}
								lowercase__       			=      3
								for i in range(last - first - min_width    ):
												if all(spm_ids[first + i + j] in removable_tokens for j in range(lowerCamelCase_    )    ):
																lowercase__       			=      [
																    k
																    for k in range(last - first - min_width    )
																    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
																]
																for j in possible_matches:
																				if check_diff(spm_ids[first : first + i]		,       tok_ids[first : first + j]		,       lowerCamelCase_		,       lowerCamelCase_    ) and check_details(
																				    lowerCamelCase_		,       spm_ids[first + i : last]		,       tok_ids[first + j : last]		,       lowerCamelCase_		,       lowerCamelCase_		,       ):
																								return True

				print(F"""Spm: {[fast.decode([spm_ids[i]]    ) for i in range(lowerCamelCase_		,       lowerCamelCase_    )]}"""    )
				try:
								print(F"""Tok: {[fast.decode([tok_ids[i]]    ) for i in range(lowerCamelCase_		,       lowerCamelCase_    )]}"""    )
				except Exception:
								pass

				lowercase__       			=      fast.decode(spm_ids[:first]    )
				lowercase__       			=      fast.decode(spm_ids[last:]    )
				lowercase__       			=      fast.decode(spm_ids[first:last]    )
				print()
				print(lowerCamelCase_    )
				return False






def   a      (						lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				global perfect
				global imperfect
				global wrong
				global total

				lowercase__       			=      slow.encode(lowerCamelCase_    )
				lowercase__       			=      fast.encode(lowerCamelCase_    )

				lowercase__       			=      False
				total += 1

				if slow_ids != fast_ids:
								if check_details(lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    ):
												lowercase__       			=      True
												imperfect += 1
								else:
												wrong += 1
				else:
								perfect += 1

				if total % 1_0000 == 0:
								print(F"""({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})"""    )

				if skip_assert:
								return

				assert (
				    slow_ids == fast_ids
				), F"""line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(lowerCamelCase_    )}\n{fast.tokenize(lowerCamelCase_    )}"""






def   a      (						lowerCamelCase_		,       lowerCamelCase_    ):






				'''simple docstring'''
				global batch_total
				for i in range(len(lowerCamelCase_    )    ):
								# premise, all languages
								for text in dataset[i]["premise"].values():
												test_string(lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    )

								# hypothesis, all languages
								for text in dataset[i]["hypothesis"]["translation"]:
												test_string(lowerCamelCase_		,       lowerCamelCase_		,       lowerCamelCase_    )


if __name__ == "__main__":
							for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
														A__     :      Optional[Any]             =			list(slow_class.max_model_input_sizes.keys())
														for checkpoint in checkpoint_names:
																					A__     :      Dict             =			0
																					A__     :      Dict             =			0
																					A__     :      List[str]             =			0
																					A__     :      List[str]             =			0

																					print(F"========================== Checking {name}: {checkpoint} ==========================")
																					A__     :      Tuple             =			slow_class.from_pretrained(checkpoint, force_download=True)
																					A__     :      Dict             =			fast_class.from_pretrained(checkpoint, force_download=True)
																					test_tokenizer(slow, fast)
																					print(F"Accuracy {perfect * 1_00 / total:.2f}")












from collections import defaultdict






def   a      (						lowerCamelCase_ = 5    ):






				'''simple docstring'''
				lowercase__       			=      defaultdict(lowerCamelCase_    )
				lowercase__       			=      0

				while True:
								lowercase__       			=      get_digits(lowerCamelCase_    )
								freqs[digits].append(lowerCamelCase_    )

								if len(freqs[digits]    ) == max_base:
												lowercase__       			=      freqs[digits][0] ** 3
												return base

								num += 1






def   a      (						lowerCamelCase_    ):






				'''simple docstring'''
				return "".join(sorted(str(num**3    )    )    )


if __name__ == "__main__":
							print(F"{solution() = }")




