





'''simple docstring'''
from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

a__							:							int			={
    name: (getattr(transformers, name), getattr(transformers, name + '''Fast''')) for name in SLOW_TO_FAST_CONVERTERS
}

a__							:							str			=datasets.load_dataset('''xnli''', split='''test+validation''')

a__							:							List[str]			=0
a__							:							List[str]			=0
a__							:							int			=0
a__							:							Any			=0
def     lowercase__       (       __lowercase						:  str    ,  __lowercase						:  Dict    ,  __lowercase						:  Union[str, Any]    ,  __lowercase						:  Optional[int]					)			->				Optional[int]:



       """simple docstring"""






       if spm_diff == list(reversed(__lowercase					)					):
              # AAA -> AA+A vs A+AA case.
              return True
       elif len(__lowercase					) == len(__lowercase					) and fast.decode(__lowercase					) == fast.decode(__lowercase					):
              # Second order OK
              # Barrich -> Barr + ich vs Bar + rich
              return True
       __UpperCamelCase	=  slow.encode(slow.decode(__lowercase					)					)
       __UpperCamelCase	=  fast.encode(fast.decode(__lowercase					)					)
       if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
              # Type 3 error.
              # Snehagatha ->
              #       Sne, h, aga, th, a
              #       Sne, ha, gat, ha
              # Encoding the wrong with sp does not even recover what spm gave us
              # It fits tokenizer however...
              return True
       return False
def     lowercase__       (       __lowercase						:  Tuple    ,  __lowercase						:  Any    ,  __lowercase						:  Optional[int]					)			->				Union[str, Any]:



       """simple docstring"""






       __UpperCamelCase	=  fast.encode_plus(__lowercase					)[0]
       __UpperCamelCase	=  enc.offsets
       __UpperCamelCase	,						__UpperCamelCase	=  offsets[idx], offsets[idx - 1]
       if curr is not None and line[curr[0] : curr[1]] == "\u200f":
              return True
       if prev is not None and line[prev[0] : prev[1]] == "\u200f":
              return True
def     lowercase__       (       __lowercase						:  Optional[int]    ,  __lowercase						:  int    ,  __lowercase						:  List[Any]    ,  __lowercase						:  int    ,  __lowercase						:  str					)			->				List[str]:



       """simple docstring"""






       for i, (spm_id, tok_id) in enumerate(zip(__lowercase    ,  __lowercase					)					):
              if spm_id != tok_id:
                     break
       __UpperCamelCase	=  i
       for i, (spm_id, tok_id) in enumerate(zip(reversed(__lowercase					)    ,  reversed(__lowercase					)					)					):
              if spm_id != tok_id:
                     break
       __UpperCamelCase	=  len(__lowercase					) - i

       __UpperCamelCase	=  spm_ids[first:last]
       __UpperCamelCase	=  tok_ids[first:last]

       if check_diff(__lowercase    ,  __lowercase    ,  __lowercase    ,  __lowercase					):
              return True

       if check_LTR_mark(__lowercase    ,  __lowercase    ,  __lowercase					):
              return True

       if last - first > 5:
              # We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
              __UpperCamelCase	=  Counter(spm_ids[first:last]					)
              __UpperCamelCase	=  Counter(tok_ids[first:last]					)

              __UpperCamelCase	=  {spm_ for (spm_, si) in spms.items() if toks.get(spm_    ,  0					) == si}
              __UpperCamelCase	=  3
              for i in range(last - first - min_width					):
                     if all(spm_ids[first + i + j] in removable_tokens for j in range(__lowercase					)					):
                            __UpperCamelCase	=  [
                                k
                                for k in range(last - first - min_width					)
                                if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
                            ]
                            for j in possible_matches:
                                   if check_diff(spm_ids[first : first + i]    ,  tok_ids[first : first + j]    ,  __lowercase    ,  __lowercase					) and check_details(
                                       __lowercase    ,  spm_ids[first + i : last]    ,  tok_ids[first + j : last]    ,  __lowercase    ,  __lowercase    ,  ):
                                          return True

       print(F'''Spm: {[fast.decode([spm_ids[i]]					) for i in range(__lowercase    ,  __lowercase					)]}'''					)
       try:
              print(F'''Tok: {[fast.decode([tok_ids[i]]					) for i in range(__lowercase    ,  __lowercase					)]}'''					)
       except Exception:
              pass

       __UpperCamelCase	=  fast.decode(spm_ids[:first]					)
       __UpperCamelCase	=  fast.decode(spm_ids[last:]					)
       __UpperCamelCase	=  fast.decode(spm_ids[first:last]					)
       print()
       print(__lowercase					)
       return False
def     lowercase__       (       __lowercase						:  Dict    ,  __lowercase						:  List[Any]    ,  __lowercase						:  Union[str, Any]					)			->				Union[str, Any]:



       """simple docstring"""






       global perfect
       global imperfect
       global wrong
       global total

       __UpperCamelCase	=  slow.encode(__lowercase					)
       __UpperCamelCase	=  fast.encode(__lowercase					)

       __UpperCamelCase	=  False
       total += 1

       if slow_ids != fast_ids:
              if check_details(__lowercase    ,  __lowercase    ,  __lowercase    ,  __lowercase    ,  __lowercase					):
                     __UpperCamelCase	=  True
                     imperfect += 1
              else:
                     wrong += 1
       else:
              perfect += 1

       if total % 10000 == 0:
              print(F'''({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})'''					)

       if skip_assert:
              return

       assert (
           slow_ids == fast_ids
       ), F'''line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(__lowercase					)}\n{fast.tokenize(__lowercase					)}'''
def     lowercase__       (       __lowercase						:  List[Any]    ,  __lowercase						:  int					)			->				int:



       """simple docstring"""






       global batch_total
       for i in range(len(__lowercase					)					):
              # premise, all languages
              for text in dataset[i]["premise"].values():
                     test_string(__lowercase    ,  __lowercase    ,  __lowercase					)

              # hypothesis, all languages
              for text in dataset[i]["hypothesis"]["translation"]:
                     test_string(__lowercase    ,  __lowercase    ,  __lowercase					)


if __name__ == "__main__":
    for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
        a__							:							int			=list(slow_class.max_model_input_sizes.keys())
        for checkpoint in checkpoint_names:
            a__							:							List[str]			=0
            a__							:							Any			=0
            a__							:							Any			=0
            a__							:							List[Any]			=0

            print(f'========================== Checking {name}: {checkpoint} ==========================')
            a__							:							Optional[int]			=slow_class.from_pretrained(checkpoint, force_download=True)
            a__							:							List[str]			=fast_class.from_pretrained(checkpoint, force_download=True)
            test_tokenizer(slow, fast)
            print(f'Accuracy {perfect * 100 / total:.2f}')







'''simple docstring'''
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


a__							:							Optional[int]			=threading.Lock()
a__							:							Optional[logging.Handler]			=None

a__							:							Optional[Any]			={
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

a__							:							List[Any]			=logging.WARNING

a__							:							str			=True
def     lowercase__       (       )			->				int:



       """simple docstring"""






       __UpperCamelCase	=  os.getenv('DIFFUSERS_VERBOSITY'    ,  __lowercase					)
       if env_level_str:
              if env_level_str in log_levels:
                     return log_levels[env_level_str]
              else:
                     logging.getLogger().warning(
                         F'''Unknown option DIFFUSERS_VERBOSITY={env_level_str}, '''
                         F'''has to be one of: { ', '.join(log_levels.keys()					) }'''					)
       return _default_log_level
def     lowercase__       (       )			->				str:



       """simple docstring"""






       return __name__.split('.'					)[0]
def     lowercase__       (       )			->				logging.Logger:



       """simple docstring"""






       return logging.getLogger(_get_library_name()					)
def     lowercase__       (       )			->				None:



       """simple docstring"""






       global _default_handler

       with _lock:
              if _default_handler:
                     # This library has already configured the library root logger.
                     return
              __UpperCamelCase	=  logging.StreamHandler()  # Set sys.stderr as stream.
              __UpperCamelCase	=  sys.stderr.flush

              # Apply our default configuration to the library root logger.
              __UpperCamelCase	=  _get_library_root_logger()
              library_root_logger.addHandler(_default_handler					)
              library_root_logger.setLevel(_get_default_logging_level()					)
              __UpperCamelCase	=  False
def     lowercase__       (       )			->				None:



       """simple docstring"""






       global _default_handler

       with _lock:
              if not _default_handler:
                     return

              __UpperCamelCase	=  _get_library_root_logger()
              library_root_logger.removeHandler(_default_handler					)
              library_root_logger.setLevel(logging.NOTSET					)
              __UpperCamelCase	=  None
def     lowercase__       (       )			->				str:



       """simple docstring"""






       return log_levels
def     lowercase__       (       __lowercase						:  Optional[str] = None					)			->				logging.Logger:



       """simple docstring"""






       if name is None:
              __UpperCamelCase	=  _get_library_name()

       _configure_library_root_logger()
       return logging.getLogger(__lowercase					)
def     lowercase__       (       )			->				int:



       """simple docstring"""






       _configure_library_root_logger()
       return _get_library_root_logger().getEffectiveLevel()
def     lowercase__       (       __lowercase						:  int					)			->				None:



       """simple docstring"""






       _configure_library_root_logger()
       _get_library_root_logger().setLevel(__lowercase					)
def     lowercase__       (       )			->				Tuple:



       """simple docstring"""






       return set_verbosity(__lowercase					)
def     lowercase__       (       )			->				Union[str, Any]:



       """simple docstring"""






       return set_verbosity(__lowercase					)
def     lowercase__       (       )			->				Any:



       """simple docstring"""






       return set_verbosity(__lowercase					)
def     lowercase__       (       )			->				int:



       """simple docstring"""






       return set_verbosity(__lowercase					)
def     lowercase__       (       )			->				None:



       """simple docstring"""






       _configure_library_root_logger()

       assert _default_handler is not None
       _get_library_root_logger().removeHandler(_default_handler					)
def     lowercase__       (       )			->				None:



       """simple docstring"""






       _configure_library_root_logger()

       assert _default_handler is not None
       _get_library_root_logger().addHandler(_default_handler					)
def     lowercase__       (       __lowercase						:  logging.Handler					)			->				None:



       """simple docstring"""






       _configure_library_root_logger()

       assert handler is not None
       _get_library_root_logger().addHandler(__lowercase					)
def     lowercase__       (       __lowercase						:  logging.Handler					)			->				None:



       """simple docstring"""






       _configure_library_root_logger()

       assert handler is not None and handler not in _get_library_root_logger().handlers
       _get_library_root_logger().removeHandler(__lowercase					)
def     lowercase__       (       )			->				None:



       """simple docstring"""






       _configure_library_root_logger()
       __UpperCamelCase	=  False
def     lowercase__       (       )			->				None:



       """simple docstring"""






       _configure_library_root_logger()
       __UpperCamelCase	=  True
def     lowercase__       (       )			->				None:



       """simple docstring"""






       __UpperCamelCase	=  _get_library_root_logger().handlers

       for handler in handlers:
              __UpperCamelCase	=  logging.Formatter('[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s'					)
              handler.setFormatter(__lowercase					)
def     lowercase__       (       )			->				None:



       """simple docstring"""






       __UpperCamelCase	=  _get_library_root_logger().handlers

       for handler in handlers:
              handler.setFormatter(__lowercase					)
def     lowercase__       (       self						:  int    ,  *__lowercase						:  List[str]    ,  **__lowercase						:  List[Any]					)			->				Any:



       """simple docstring"""






       __UpperCamelCase	=  os.getenv('DIFFUSERS_NO_ADVISORY_WARNINGS'    ,  __lowercase					)
       if no_advisory_warnings:
              return
       self.warning(*__lowercase    ,  **__lowercase					)


a__							:							Dict			=warning_advice

class 					snake_case					:




 """simple docstring"""



 def __init__(	self	:							Optional[int]							,     *__A	:							int							,     **__A	:							Dict     ):  # pylint: disable=unused-argument
        __UpperCamelCase	=  args[0] if args else None






 def __iter__(	self	:							str     ):
        return iter(self._iterator     )






 def __getattr__(	self	:							Tuple							,     __A	:							Any     ):

        def empty_fn(*__A	:							Any							,     **__A	:							List[str]     ):  # pylint: disable=unused-argument
               return

        return empty_fn






 def __enter__(	self	:							str     ):
        return self






 def __exit__(	self	:							Optional[Any]							,     __A	:							Dict							,     __A	:							str							,     __A	:							Dict     ):
        return






class 					snake_case					:




 """simple docstring"""



 def __call__(	self	:							Union[str, Any]							,     *__A	:							Union[str, Any]							,     **__A	:							Dict     ):
        if _tqdm_active:
               return tqdm_lib.tqdm(*__A							,     **__A     )
        else:
               return EmptyTqdm(*__A							,     **__A     )






 def 	_lowerCamelCase (	self	:							Any							,     *__A	:							str							,     **__A	:							Optional[int]     ):
        __UpperCamelCase	=  None
        if _tqdm_active:
               return tqdm_lib.tqdm.set_lock(*__A							,     **__A     )






 def 	_lowerCamelCase (	self	:							Optional[int]     ):
        if _tqdm_active:
               return tqdm_lib.tqdm.get_lock()






a__							:							List[Any]			=_tqdm_cls()
def     lowercase__       (       )			->				bool:



       """simple docstring"""






       global _tqdm_active
       return bool(_tqdm_active					)
def     lowercase__       (       )			->				List[str]:



       """simple docstring"""






       global _tqdm_active
       __UpperCamelCase	=  True


def     lowercase__       (       )			->				str:



       """simple docstring"""






       global _tqdm_active
       __UpperCamelCase	=  False
