
import argparse
import collections
import importlib.util
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
lowercase_											=	"""src/diffusers"""
lowercase_											=	"""docs/source/en"""
lowercase_											=	"""."""


def     a__			(    snake_case   ,	snake_case   ,	snake_case    ):





  """simple docstring"""
  with open(snake_case   ,	'''r'''   ,	encoding='''utf-8'''   ,	newline='''\n'''    ) as f:
    __SCREAMING_SNAKE_CASE     :					List[Any]			=						f.readlines()
  # Find the start prompt.
  __SCREAMING_SNAKE_CASE     :					Any			=						0
  while not lines[start_index].startswith(snake_case    ):
    start_index += 1
  start_index += 1

  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						start_index
  while not lines[end_index].startswith(snake_case    ):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]    ) <= 1:
    start_index += 1
  while len(lines[end_index]    ) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]    ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
lowercase_											=	"""Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
lowercase_											=	re.compile(R"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
lowercase_											=	re.compile(R"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowercase_											=	re.compile(R"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the diffusers module imported is the one in the repo.
lowercase_											=	importlib.util.spec_from_file_location(
    """diffusers""",
    os.path.join(TRANSFORMERS_PATH, """__init__.py"""),
    submodule_search_locations=[TRANSFORMERS_PATH],
)
lowercase_											=	spec.loader.load_module()


def     a__			(    snake_case    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					List[Any]			=						re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''   ,	snake_case    )
  return [m.group(0    ) for m in matches]


def     a__			(    snake_case   ,	snake_case    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						2 if text == '''✅''' or text == '''❌''' else len(snake_case    )
  __SCREAMING_SNAKE_CASE     :					Optional[int]			=						(width - text_length) // 2
  __SCREAMING_SNAKE_CASE     :					Tuple			=						width - text_length - left_indent
  return " " * left_indent + text + " " * right_indent


def     a__			(    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE     :					Dict			=						diffusers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
  __SCREAMING_SNAKE_CASE     :					int			=						{
      name: config_mapping_names[code]
      for code, name in diffusers_module.MODEL_NAMES_MAPPING.items()
      if code in config_mapping_names
  }
  __SCREAMING_SNAKE_CASE     :					str			=						{name: config.replace('''ConfigMixin'''   ,	''''''    ) for name, config in model_name_to_config.items()}

  # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
  __SCREAMING_SNAKE_CASE     :					List[str]			=						collections.defaultdict(snake_case    )
  __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						collections.defaultdict(snake_case    )
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						collections.defaultdict(snake_case    )
  __SCREAMING_SNAKE_CASE     :					List[Any]			=						collections.defaultdict(snake_case    )
  __SCREAMING_SNAKE_CASE     :					Tuple			=						collections.defaultdict(snake_case    )

  # Let's lookup through all diffusers object (once).
  for attr_name in dir(snake_case    ):
    __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						None
    if attr_name.endswith('''Tokenizer'''    ):
      __SCREAMING_SNAKE_CASE     :					str			=						slow_tokenizers
      __SCREAMING_SNAKE_CASE     :					str			=						attr_name[:-9]
    elif attr_name.endswith('''TokenizerFast'''    ):
      __SCREAMING_SNAKE_CASE     :					Optional[int]			=						fast_tokenizers
      __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						attr_name[:-13]
    elif _re_tf_models.match(snake_case    ) is not None:
      __SCREAMING_SNAKE_CASE     :					Union[str, Any]			=						tf_models
      __SCREAMING_SNAKE_CASE     :					Tuple			=						_re_tf_models.match(snake_case    ).groups()[0]
    elif _re_flax_models.match(snake_case    ) is not None:
      __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						flax_models
      __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						_re_flax_models.match(snake_case    ).groups()[0]
    elif _re_pt_models.match(snake_case    ) is not None:
      __SCREAMING_SNAKE_CASE     :					List[Any]			=						pt_models
      __SCREAMING_SNAKE_CASE     :					int			=						_re_pt_models.match(snake_case    ).groups()[0]

    if lookup_dict is not None:
      while len(snake_case    ) > 0:
        if attr_name in model_name_to_prefix.values():
          __SCREAMING_SNAKE_CASE     :					int			=						True
          break
        # Try again after removing the last word in the name
        __SCREAMING_SNAKE_CASE     :					Tuple			=						''''''.join(camel_case_split(snake_case    )[:-1]    )

    # Let's build that table!
  __SCREAMING_SNAKE_CASE     :					Optional[int]			=						list(model_name_to_config.keys()    )
  model_names.sort(key=str.lower    )
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						['''Model''', '''Tokenizer slow''', '''Tokenizer fast''', '''PyTorch support''', '''TensorFlow support''', '''Flax Support''']
  # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						[len(snake_case    ) + 2 for c in columns]
  __SCREAMING_SNAKE_CASE     :					Tuple			=						max([len(snake_case    ) for name in model_names]    ) + 2

  # Build the table per se
  __SCREAMING_SNAKE_CASE     :					List[Any]			=						'''|''' + '''|'''.join([_center_text(snake_case   ,	snake_case    ) for c, w in zip(snake_case   ,	snake_case    )]    ) + '''|\n'''
  # Use ":-----:" format to center-aligned table cell texts
  table += "|" + "|".join([''':''' + '''-''' * (w - 2) + ''':''' for w in widths]    ) + "|\n"

  __SCREAMING_SNAKE_CASE     :					Any			=						{True: '''✅''', False: '''❌'''}
  for name in model_names:
    __SCREAMING_SNAKE_CASE     :					str			=						model_name_to_prefix[name]
    __SCREAMING_SNAKE_CASE     :					int			=						[
        name,
        check[slow_tokenizers[prefix]],
        check[fast_tokenizers[prefix]],
        check[pt_models[prefix]],
        check[tf_models[prefix]],
        check[flax_models[prefix]],
    ]
    table += "|" + "|".join([_center_text(snake_case   ,	snake_case    ) for l, w in zip(snake_case   ,	snake_case    )]    ) + "|\n"
  return table


def     a__			(    snake_case=False    ):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE,						__SCREAMING_SNAKE_CASE,						__SCREAMING_SNAKE_CASE,						__SCREAMING_SNAKE_CASE     :					List[str]			=						_find_text_in_file(
      filename=os.path.join(snake_case   ,	'''index.mdx'''    )   ,	start_prompt='''<!--This table is updated automatically from the auto modules'''   ,	end_prompt='''<!-- End table-->'''   ,	)
  __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						get_model_table_from_auto_modules()

  if current_table != new_table:
    if overwrite:
      with open(os.path.join(snake_case   ,	'''index.mdx'''    )   ,	'''w'''   ,	encoding='''utf-8'''   ,	newline='''\n'''    ) as f:
        f.writelines(lines[:start_index] + [new_table] + lines[end_index:]    )
    else:
      raise ValueError(
          '''The model table in the `index.mdx` has not been updated. Run `make fix-copies` to fix this.'''    )


if __name__ == "__main__":
   lowercase_											=	argparse.ArgumentParser()
   parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
   lowercase_											=	parser.parse_args()

   check_model_table(args.fix_and_overwrite)


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowercase_											=	logging.get_logger(__name__)

lowercase_											=	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase_											=	{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

lowercase_											=	{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

lowercase_											=	{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}





class   __UpperCamelCase				(    lowerCAmelCase__   ):






     """simple docstring"""







     lowerCAmelCase_     =	VOCAB_FILES_NAMES
     lowerCAmelCase_     =	PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_     =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_     =	['''input_ids''', '''attention_mask''', '''token_type_ids''']
     lowerCAmelCase_     =	DebertaTokenizer



     def __init__(							self    :    List[str]      ,		_A    :    Tuple=None      ,		_A    :    List[Any]=None      ,		_A    :    Optional[int]=None      ,		_A    :    List[str]="replace"      ,		_A    :    List[Any]="[CLS]"      ,		_A    :    Optional[Any]="[SEP]"      ,		_A    :    Union[str, Any]="[SEP]"      ,		_A    :    List[str]="[CLS]"      ,		_A    :    Optional[Any]="[UNK]"      ,		_A    :    str="[PAD]"      ,		_A    :    List[str]="[MASK]"      ,		_A    :    Dict=False      ,		**_A    :    Optional[int]      ,		):







       """simple docstring"""




       super().__init__(
           _A      ,		_A      ,		tokenizer_file=_A      ,		errors=_A      ,		bos_token=_A      ,		eos_token=_A      ,		unk_token=_A      ,		sep_token=_A      ,		cls_token=_A      ,		pad_token=_A      ,		mask_token=_A      ,		add_prefix_space=_A      ,		**_A      ,		)
       __SCREAMING_SNAKE_CASE     :					List[str]			=						kwargs.pop('''add_bos_token'''      ,		_A				)

       __SCREAMING_SNAKE_CASE     :					int			=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
       if pre_tok_state.get('''add_prefix_space'''      ,		_A				) != add_prefix_space:
         __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						getattr(_A      ,		pre_tok_state.pop('''type'''				)				)
         __SCREAMING_SNAKE_CASE     :					Dict			=						add_prefix_space
         __SCREAMING_SNAKE_CASE     :					Any			=						pre_tok_class(**_A				)

       __SCREAMING_SNAKE_CASE     :					Optional[Any]			=						add_prefix_space



     @property
     def 				UpperCAmelCase__    (							self    :    List[Any]				):







       """simple docstring"""




       if self._mask_token is None:
         if self.verbose:
           logger.error('''Using mask_token, but it is not set yet.'''				)
         return None
       return str(self._mask_token				)



     @mask_token.setter
     def 				UpperCAmelCase__    (							self    :    Any      ,		_A    :    Union[str, Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Dict			=						AddedToken(_A      ,		lstrip=_A      ,		rstrip=_A				) if isinstance(_A      ,		_A				) else value
       __SCREAMING_SNAKE_CASE     :					Tuple			=						value



     def 				UpperCAmelCase__    (							self    :    List[Any]      ,		_A    :    List[int]      ,		_A    :    Optional[List[int]] = None				):







       """simple docstring"""




       if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       __SCREAMING_SNAKE_CASE     :					List[str]			=						[self.cls_token_id]
       __SCREAMING_SNAKE_CASE     :					List[Any]			=						[self.sep_token_id]
       return cls + token_ids_a + sep + token_ids_a + sep



     def 				UpperCAmelCase__    (							self    :    int      ,		_A    :    List[int]      ,		_A    :    Optional[List[int]] = None				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Tuple			=						[self.sep_token_id]
       __SCREAMING_SNAKE_CASE     :					List[str]			=						[self.cls_token_id]

       if token_ids_a is None:
         return len(cls + token_ids_a + sep				) * [0]
       return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		*_A    :    Optional[Any]      ,		**_A    :    Optional[Any]				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					List[str]			=						kwargs.get('''is_split_into_words'''      ,		_A				)
       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*_A      ,		**_A				)



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		*_A    :    Tuple      ,		**_A    :    int				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Dict			=						kwargs.get('''is_split_into_words'''      ,		_A				)

       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*_A      ,		**_A				)



     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    str      ,		_A    :    Optional[str] = None				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					Optional[int]			=						self._tokenizer.model.save(_A      ,		name=_A				)
       return tuple(_A				)




     def 				UpperCAmelCase__    (							self    :    Tuple      ,		_A    :    "Conversation"				):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE     :					List[Any]			=						[]
       for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(_A      ,		add_special_tokens=_A				) + [self.eos_token_id]				)

       if len(_A				) > self.model_max_length:
         __SCREAMING_SNAKE_CASE     :					Any			=						input_ids[-self.model_max_length :]
       return input_ids
