




from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


lowerCamelCase_  						=       logging.get_logger(__name__)

lowerCamelCase_  						=       {
    '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json''',
    # See all MBART models at https://huggingface.co/models?filter=mbart
}
class 				__a (			UpperCamelCase__					):


     """simple docstring"""


     _A    :   str     =  "mbart"
     _A    :   Any     =  ["past_key_values"]
     _A    :   Tuple     =  {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}




     def __init__(     self      :			Tuple    ,_UpperCamelCase      :			Optional[Any]=5_0_2_6_5    ,_UpperCamelCase      :			Dict=1_0_2_4    ,_UpperCamelCase      :			Dict=1_2    ,_UpperCamelCase      :			int=4_0_9_6    ,_UpperCamelCase      :			List[Any]=1_6    ,_UpperCamelCase      :			List[Any]=1_2    ,_UpperCamelCase      :			int=4_0_9_6    ,_UpperCamelCase      :			Any=1_6    ,_UpperCamelCase      :			List[str]=0.0    ,_UpperCamelCase      :			Union[str, Any]=0.0    ,_UpperCamelCase      :			Optional[int]=True    ,_UpperCamelCase      :			Optional[int]=True    ,_UpperCamelCase      :			Union[str, Any]="gelu"    ,_UpperCamelCase      :			Any=1_0_2_4    ,_UpperCamelCase      :			Dict=0.1    ,_UpperCamelCase      :			Dict=0.0    ,_UpperCamelCase      :			Optional[int]=0.0    ,_UpperCamelCase      :			Tuple=0.02    ,_UpperCamelCase      :			List[Any]=0.0    ,_UpperCamelCase      :			List[Any]=False    ,_UpperCamelCase      :			int=1    ,_UpperCamelCase      :			Optional[Any]=0    ,_UpperCamelCase      :			Dict=2    ,_UpperCamelCase      :			int=2    ,**_UpperCamelCase      :			Union[str, Any]    ,)		->       Union[str, Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =vocab_size
         SCREAMING_SNAKE_CASE__       =max_position_embeddings
         SCREAMING_SNAKE_CASE__       =d_model
         SCREAMING_SNAKE_CASE__       =encoder_ffn_dim
         SCREAMING_SNAKE_CASE__       =encoder_layers
         SCREAMING_SNAKE_CASE__       =encoder_attention_heads
         SCREAMING_SNAKE_CASE__       =decoder_ffn_dim
         SCREAMING_SNAKE_CASE__       =decoder_layers
         SCREAMING_SNAKE_CASE__       =decoder_attention_heads
         SCREAMING_SNAKE_CASE__       =dropout
         SCREAMING_SNAKE_CASE__       =attention_dropout
         SCREAMING_SNAKE_CASE__       =activation_dropout
         SCREAMING_SNAKE_CASE__       =activation_function
         SCREAMING_SNAKE_CASE__       =init_std
         SCREAMING_SNAKE_CASE__       =encoder_layerdrop
         SCREAMING_SNAKE_CASE__       =decoder_layerdrop
         SCREAMING_SNAKE_CASE__       =classifier_dropout
         SCREAMING_SNAKE_CASE__       =use_cache
         SCREAMING_SNAKE_CASE__       =encoder_layers
         SCREAMING_SNAKE_CASE__       =scale_embedding  # scale factor will be sqrt(d_model) if True
         super().__init__(
             pad_token_id=__lowerCamelCase    ,bos_token_id=__lowerCamelCase    ,eos_token_id=__lowerCamelCase    ,is_encoder_decoder=__lowerCamelCase    ,forced_eos_token_id=__lowerCamelCase    ,**__lowerCamelCase    ,)






class 				__a (			UpperCamelCase__					):


     """simple docstring"""


     @property
     def   __A    (     self      :			Union[str, Any]						)		->       List[str]:


         '''simple docstring'''





         if self.task in ["default", "seq2seq-lm"]:
             SCREAMING_SNAKE_CASE__       =OrderedDict(
                 [
                     ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                 ]						)

             if self.use_past:
                 SCREAMING_SNAKE_CASE__       ={0: "batch"}
                 SCREAMING_SNAKE_CASE__       ={0: "batch", 1: "past_decoder_sequence + sequence"}
             else:
                 SCREAMING_SNAKE_CASE__       ={0: "batch", 1: "decoder_sequence"}
                 SCREAMING_SNAKE_CASE__       ={0: "batch", 1: "decoder_sequence"}

             if self.use_past:
                 self.fill_with_past_key_values_(__lowerCamelCase    ,direction="""inputs"""						)
         elif self.task == "causal-lm":
             # TODO: figure this case out.
             SCREAMING_SNAKE_CASE__       =OrderedDict(
                 [
                     ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                 ]						)
             if self.use_past:
                 SCREAMING_SNAKE_CASE__       =self.num_layers
                 for i in range(__lowerCamelCase						):
                     SCREAMING_SNAKE_CASE__       ={0: "batch", 2: "past_sequence + sequence"}
                     SCREAMING_SNAKE_CASE__       ={0: "batch", 2: "past_sequence + sequence"}
         else:
             SCREAMING_SNAKE_CASE__       =OrderedDict(
                 [
                     ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
                     ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
                     ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
                 ]						)

         return common_inputs




     @property
     def   __A    (     self      :			Tuple						)		->       Tuple:


         '''simple docstring'''





         if self.task in ["default", "seq2seq-lm"]:
             SCREAMING_SNAKE_CASE__       =super().outputs
         else:
             SCREAMING_SNAKE_CASE__       =super(__lowerCamelCase    ,self						).outputs
             if self.use_past:
                 SCREAMING_SNAKE_CASE__       =self.num_layers
                 for i in range(__lowerCamelCase						):
                     SCREAMING_SNAKE_CASE__       ={0: "batch", 2: "past_sequence + sequence"}
                     SCREAMING_SNAKE_CASE__       ={0: "batch", 2: "past_sequence + sequence"}
         return common_outputs




     def   __A    (     self      :			Any    ,_UpperCamelCase      :			PreTrainedTokenizer    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			bool = False    ,_UpperCamelCase      :			Optional[TensorType] = None    ,)		->       List[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             __lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase						)

         # Generate decoder inputs
         SCREAMING_SNAKE_CASE__       =seq_length if not self.use_past else 1
         SCREAMING_SNAKE_CASE__       =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             __lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase						)
         SCREAMING_SNAKE_CASE__       ={f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
         SCREAMING_SNAKE_CASE__       =dict(**__lowerCamelCase    ,**__lowerCamelCase						)

         if self.use_past:
             if not is_torch_available():
                 raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""						)
             else:
                 import torch
             SCREAMING_SNAKE_CASE__       =common_inputs["input_ids"].shape
             SCREAMING_SNAKE_CASE__       =common_inputs["decoder_input_ids"].shape[1]
             SCREAMING_SNAKE_CASE__       =self.num_attention_heads
             SCREAMING_SNAKE_CASE__       =(
                 batch,
                 num_encoder_attention_heads,
                 encoder_seq_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )
             SCREAMING_SNAKE_CASE__       =decoder_seq_length + 3
             SCREAMING_SNAKE_CASE__       =(
                 batch,
                 num_decoder_attention_heads,
                 decoder_past_length,
                 self._config.hidden_size // num_decoder_attention_heads,
             )

             SCREAMING_SNAKE_CASE__       =torch.cat(
                 [common_inputs["""decoder_attention_mask"""], torch.ones(__lowerCamelCase    ,__lowerCamelCase						)]    ,dim=1						)

             SCREAMING_SNAKE_CASE__       =[]
             # If the number of encoder and decoder layers are present in the model configuration, both are considered
             SCREAMING_SNAKE_CASE__       =self.num_layers
             SCREAMING_SNAKE_CASE__       =min(__lowerCamelCase    ,__lowerCamelCase						)
             SCREAMING_SNAKE_CASE__       =max(__lowerCamelCase    ,__lowerCamelCase						) - min_num_layers
             SCREAMING_SNAKE_CASE__       ="encoder" if num_encoder_layers > num_decoder_layers else "decoder"

             for _ in range(__lowerCamelCase						):
                 common_inputs["past_key_values"].append(
                     (
                         torch.zeros(__lowerCamelCase						),
                         torch.zeros(__lowerCamelCase						),
                         torch.zeros(__lowerCamelCase						),
                         torch.zeros(__lowerCamelCase						),
                     )						)
             # TODO: test this.
             SCREAMING_SNAKE_CASE__       =encoder_shape if remaining_side_name == "encoder" else decoder_shape
             for _ in range(__lowerCamelCase    ,__lowerCamelCase						):
                 common_inputs["past_key_values"].append((torch.zeros(__lowerCamelCase						), torch.zeros(__lowerCamelCase						))						)
         return common_inputs




     def   __A    (     self      :			Any    ,_UpperCamelCase      :			PreTrainedTokenizer    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			bool = False    ,_UpperCamelCase      :			Optional[TensorType] = None    ,)		->       Optional[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             __lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase						)

         if self.use_past:
             if not is_torch_available():
                 raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""						)
             else:
                 import torch
             SCREAMING_SNAKE_CASE__       =common_inputs["input_ids"].shape
             # Not using the same length for past_key_values
             SCREAMING_SNAKE_CASE__       =seqlen + 2
             SCREAMING_SNAKE_CASE__       =self.num_layers
             SCREAMING_SNAKE_CASE__       =self.num_attention_heads
             SCREAMING_SNAKE_CASE__       =(
                 batch,
                 num_encoder_attention_heads,
                 past_key_values_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )

             SCREAMING_SNAKE_CASE__       =common_inputs["attention_mask"].dtype
             SCREAMING_SNAKE_CASE__       =torch.cat(
                 [common_inputs["""attention_mask"""], torch.ones(__lowerCamelCase    ,__lowerCamelCase    ,dtype=__lowerCamelCase						)]    ,dim=1						)
             SCREAMING_SNAKE_CASE__       =[
                 (torch.zeros(__lowerCamelCase						), torch.zeros(__lowerCamelCase						)) for _ in range(__lowerCamelCase						)
             ]
         return common_inputs




     def   __A    (     self      :			Dict    ,_UpperCamelCase      :			PreTrainedTokenizer    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			bool = False    ,_UpperCamelCase      :			Optional[TensorType] = None    ,)		->       Dict:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =compute_effective_axis_dimension(
             __lowerCamelCase    ,fixed_dimension=OnnxConfig.default_fixed_batch    ,num_token_to_add=0						)

         # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
         SCREAMING_SNAKE_CASE__       =tokenizer.num_special_tokens_to_add(__lowerCamelCase						)
         SCREAMING_SNAKE_CASE__       =compute_effective_axis_dimension(
             __lowerCamelCase    ,fixed_dimension=OnnxConfig.default_fixed_sequence    ,num_token_to_add=__lowerCamelCase						)

         # Generate dummy inputs according to compute batch and sequence
         SCREAMING_SNAKE_CASE__       =[" ".join([tokenizer.unk_token]						) * seq_length] * batch_size
         SCREAMING_SNAKE_CASE__       =dict(tokenizer(__lowerCamelCase    ,return_tensors=__lowerCamelCase						)						)
         return common_inputs




     def   __A    (     self      :			Any    ,_UpperCamelCase      :			PreTrainedTokenizer    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			int = -1    ,_UpperCamelCase      :			bool = False    ,_UpperCamelCase      :			Optional[TensorType] = None    ,)		->       str:


         '''simple docstring'''





         if self.task in ["default", "seq2seq-lm"]:
             SCREAMING_SNAKE_CASE__       =self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                 __lowerCamelCase    ,batch_size=__lowerCamelCase    ,seq_length=__lowerCamelCase    ,is_pair=__lowerCamelCase    ,framework=__lowerCamelCase						)

         elif self.task == "causal-lm":
             SCREAMING_SNAKE_CASE__       =self._generate_dummy_inputs_for_causal_lm(
                 __lowerCamelCase    ,batch_size=__lowerCamelCase    ,seq_length=__lowerCamelCase    ,is_pair=__lowerCamelCase    ,framework=__lowerCamelCase						)
         else:
             SCREAMING_SNAKE_CASE__       =self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 __lowerCamelCase    ,batch_size=__lowerCamelCase    ,seq_length=__lowerCamelCase    ,is_pair=__lowerCamelCase    ,framework=__lowerCamelCase						)

         return common_inputs




     def   __A    (     self      :			str    ,_UpperCamelCase      :			Optional[int]    ,_UpperCamelCase      :			List[str]    ,_UpperCamelCase      :			Union[str, Any]    ,_UpperCamelCase      :			int						)		->       str:


         '''simple docstring'''





         if self.task in ["default", "seq2seq-lm"]:
             SCREAMING_SNAKE_CASE__       =super()._flatten_past_key_values_(__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase						)
         else:
             SCREAMING_SNAKE_CASE__       =super(__lowerCamelCase    ,self						)._flatten_past_key_values_(
                 __lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase    ,__lowerCamelCase						)












import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
    from .tokenization_deberta_va import DebertaVaTokenizer
else:
    lowerCamelCase_  						=       None

lowerCamelCase_  						=       logging.get_logger(__name__)

lowerCamelCase_  						=       {"vocab_file": "spm.model", "tokenizer_file": "tokenizer.json"}

lowerCamelCase_  						=       {
    "vocab_file": {
        "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model"
        ),
        "microsoft/deberta-v2-xxlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model"
        ),
    }
}

lowerCamelCase_  						=       {
    "microsoft/deberta-v2-xlarge": 512,
    "microsoft/deberta-v2-xxlarge": 512,
    "microsoft/deberta-v2-xlarge-mnli": 512,
    "microsoft/deberta-v2-xxlarge-mnli": 512,
}

lowerCamelCase_  						=       {
    "microsoft/deberta-v2-xlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xlarge-mnli": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge-mnli": {"do_lower_case": False},
}
class 				__a (			__lowerCamelCase					):


     """simple docstring"""


     _A    :   str     =  VOCAB_FILES_NAMES
     _A    :   Union[str, Any]     =  PRETRAINED_VOCAB_FILES_MAP
     _A    :   Optional[int]     =  PRETRAINED_INIT_CONFIGURATION
     _A    :   Any     =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A    :   Optional[int]     =  DebertaVaTokenizer




     def __init__(     self      :			List[Any]    ,_UpperCamelCase      :			Tuple=None    ,_UpperCamelCase      :			str=None    ,_UpperCamelCase      :			Any=False    ,_UpperCamelCase      :			List[Any]=False    ,_UpperCamelCase      :			Any="[CLS]"    ,_UpperCamelCase      :			Optional[int]="[SEP]"    ,_UpperCamelCase      :			int="[UNK]"    ,_UpperCamelCase      :			int="[SEP]"    ,_UpperCamelCase      :			Tuple="[PAD]"    ,_UpperCamelCase      :			Optional[Any]="[CLS]"    ,_UpperCamelCase      :			Optional[int]="[MASK]"    ,**_UpperCamelCase      :			int    ,)		->       None:


         '''simple docstring'''





         super().__init__(
             _UpperCamelCase    ,tokenizer_file=_UpperCamelCase    ,do_lower_case=_UpperCamelCase    ,bos_token=_UpperCamelCase    ,eos_token=_UpperCamelCase    ,unk_token=_UpperCamelCase    ,sep_token=_UpperCamelCase    ,pad_token=_UpperCamelCase    ,cls_token=_UpperCamelCase    ,mask_token=_UpperCamelCase    ,split_by_punct=_UpperCamelCase    ,**_UpperCamelCase    ,)

         SCREAMING_SNAKE_CASE__       =do_lower_case
         SCREAMING_SNAKE_CASE__       =split_by_punct
         SCREAMING_SNAKE_CASE__       =vocab_file
         SCREAMING_SNAKE_CASE__       =False if not self.vocab_file else True




     def   __A    (     self      :			Optional[Any]    ,_UpperCamelCase      :			int    ,_UpperCamelCase      :			str=None						)		->       Dict:


         '''simple docstring'''





         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         SCREAMING_SNAKE_CASE__       =[self.cls_token_id]
         SCREAMING_SNAKE_CASE__       =[self.sep_token_id]
         return cls + token_ids_a + sep + token_ids_a + sep




     def   __A    (     self      :			List[str]    ,_UpperCamelCase      :			Any    ,_UpperCamelCase      :			List[Any]=None    ,_UpperCamelCase      :			Union[str, Any]=False						)		->       str:


         '''simple docstring'''





         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_UpperCamelCase    ,token_ids_a=_UpperCamelCase    ,already_has_special_tokens=_UpperCamelCase						)

         if token_ids_a is not None:
             return [1] + ([0] * len(_UpperCamelCase						)) + [1] + ([0] * len(_UpperCamelCase						)) + [1]
         return [1] + ([0] * len(_UpperCamelCase						)) + [1]




     def   __A    (     self      :			Union[str, Any]    ,_UpperCamelCase      :			List[str]    ,_UpperCamelCase      :			str=None						)		->       int:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__       =[self.sep_token_id]
         SCREAMING_SNAKE_CASE__       =[self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




     def   __A    (     self      :			int    ,_UpperCamelCase      :			str    ,_UpperCamelCase      :			Optional[str] = None						)		->       Tuple[str]:


         '''simple docstring'''





         if not self.can_save_slow_tokenizer:
             raise ValueError(
                 """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                 """tokenizer."""						)

         if not os.path.isdir(_UpperCamelCase						):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
             return
         SCREAMING_SNAKE_CASE__       =os.path.join(
             _UpperCamelCase    ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

         if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCamelCase						):
             copyfile(self.vocab_file    ,_UpperCamelCase						)

         return (out_vocab_file,)






