



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__lowerCamelCase								=   logging.get_logger(__name__)

__lowerCamelCase								=   {
    """google/bigbird-pegasus-large-arxiv""": (
        """https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json"""
    ),
    """google/bigbird-pegasus-large-pubmed""": (
        """https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json"""
    ),
    """google/bigbird-pegasus-large-bigpatent""": (
        """https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json"""
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}



class 						_snake_case (     lowerCamelCase			):





 """simple docstring"""



 lowerCamelCase_       =     '''bigbird_pegasus'''
 lowerCamelCase_       =     ['''past_key_values''']
 lowerCamelCase_       =     {
     '''num_attention_heads''': '''encoder_attention_heads''',
     '''hidden_size''': '''d_model''',
     '''attention_probs_dropout_prob''': '''attention_dropout''',
 }
 def __init__(  self		,      a=9_6_1_0_3		,      a=4_0_9_6		,      a=1_6		,      a=4_0_9_6		,      a=1_6		,      a=1_6		,      a=4_0_9_6		,      a=1_6		,      a=0.0		,      a=0.0		,      a=True		,      a=True		,      a="gelu_new"		,      a=1_0_2_4		,      a=0.1		,      a=0.0		,      a=0.0		,      a=0.02		,      a=2		,      a=0.0		,      a=True		,      a=0		,      a=2		,      a=1		,      a="block_sparse"		,      a=6_4		,      a=3		,      a=False		,      **a		,      )		->							List[Any]:




   """simple docstring"""







   _A          =      vocab_size
   _A          =      max_position_embeddings
   _A          =      d_model
   _A          =      encoder_ffn_dim
   _A          =      encoder_layers
   _A          =      encoder_attention_heads
   _A          =      decoder_ffn_dim
   _A          =      decoder_layers
   _A          =      decoder_attention_heads
   _A          =      dropout
   _A          =      attention_dropout
   _A          =      activation_dropout
   _A          =      activation_function
   _A          =      init_std
   _A          =      encoder_layerdrop
   _A          =      decoder_layerdrop
   _A          =      classifier_dropout
   _A          =      use_cache
   _A          =      encoder_layers
   _A          =      scale_embedding  # scale factor will be sqrt(d_model) if True

   # extra config
   _A          =      attention_type
   _A          =      block_size
   _A          =      num_random_blocks
   _A          =      use_bias

   super().__init__(
       pad_token_id=a		,      bos_token_id=a		,      eos_token_id=a		,      is_encoder_decoder=a		,      decoder_start_token_id=a		,      **a		,      )







class 						_snake_case (     lowerCamelCase			):





 """simple docstring"""



 @property
 def lowercase_						(  self   )		->							Mapping[str, Mapping[int, str]]:




   """simple docstring"""







   if self.task in ["default", "seq2seq-lm"]:
     _A          =      OrderedDict(
         [
             ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
             ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
         ]   )

     if self.use_past:
       _A          =      {0: '''batch'''}
       _A          =      {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
     else:
       _A          =      {0: '''batch''', 1: '''decoder_sequence'''}
       _A          =      {0: '''batch''', 1: '''decoder_sequence'''}

     if self.use_past:
       self.fill_with_past_key_values_(a		,      direction='''inputs'''   )
   elif self.task == "causal-lm":
     # TODO: figure this case out.
     _A          =      OrderedDict(
         [
             ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
             ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
         ]   )
     if self.use_past:
       _A      ,				_A          =      self.num_layers
       for i in range(a   ):
         _A          =      {0: '''batch''', 2: '''past_sequence + sequence'''}
         _A          =      {0: '''batch''', 2: '''past_sequence + sequence'''}
   else:
     _A          =      OrderedDict(
         [
             ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
             ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
             ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
             ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
         ]   )

   return common_inputs
 @property
 def lowercase_						(  self   )		->							Mapping[str, Mapping[int, str]]:




   """simple docstring"""







   if self.task in ["default", "seq2seq-lm"]:
     _A          =      super().outputs
   else:
     _A          =      super(a		,      self   ).outputs
     if self.use_past:
       _A      ,				_A          =      self.num_layers
       for i in range(a   ):
         _A          =      {0: '''batch''', 2: '''past_sequence + sequence'''}
         _A          =      {0: '''batch''', 2: '''past_sequence + sequence'''}
   return common_outputs
 def lowercase_						(  self		,      a		,      a = -1		,      a = -1		,      a = False		,      a = None		,      )		->							Mapping[str, Any]:




   """simple docstring"""







   _A          =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
       a		,      a		,      a		,      a		,      a   )

   # Generate decoder inputs
   _A          =      seq_length if not self.use_past else 1
   _A          =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
       a		,      a		,      a		,      a		,      a   )
   _A          =      {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
   _A          =      dict(**a		,      **a   )

   if self.use_past:
     if not is_torch_available():
       raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''   )
     else:
       import torch
     _A      ,				_A          =      common_inputs['''input_ids'''].shape
     _A          =      common_inputs['''decoder_input_ids'''].shape[1]
     _A      ,				_A          =      self.num_attention_heads
     _A          =      (
         batch,
         num_encoder_attention_heads,
         encoder_seq_length,
         self._config.hidden_size // num_encoder_attention_heads,
     )
     _A          =      decoder_seq_length + 3
     _A          =      (
         batch,
         num_decoder_attention_heads,
         decoder_past_length,
         self._config.hidden_size // num_decoder_attention_heads,
     )

     _A          =      torch.cat(
         [common_inputs['''decoder_attention_mask'''], torch.ones(a		,      a   )]		,      dim=1   )

     _A          =      []
     # If the number of encoder and decoder layers are present in the model configuration, both are considered
     _A      ,				_A          =      self.num_layers
     _A          =      min(a		,      a   )
     _A          =      max(a		,      a   ) - min_num_layers
     _A          =      '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

     for _ in range(a   ):
       common_inputs["past_key_values"].append(
           (
               torch.zeros(a   ),
               torch.zeros(a   ),
               torch.zeros(a   ),
               torch.zeros(a   ),
           )   )
     # TODO: test this.
     _A          =      encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
     for _ in range(a		,      a   ):
       common_inputs["past_key_values"].append((torch.zeros(a   ), torch.zeros(a   ))   )
   return common_inputs
 def lowercase_						(  self		,      a		,      a = -1		,      a = -1		,      a = False		,      a = None		,      )		->							Mapping[str, Any]:




   """simple docstring"""







   _A          =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
       a		,      a		,      a		,      a		,      a   )

   if self.use_past:
     if not is_torch_available():
       raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''   )
     else:
       import torch
     _A      ,				_A          =      common_inputs['''input_ids'''].shape
     # Not using the same length for past_key_values
     _A          =      seqlen + 2
     _A      ,				_A          =      self.num_layers
     _A      ,				_A          =      self.num_attention_heads
     _A          =      (
         batch,
         num_encoder_attention_heads,
         past_key_values_length,
         self._config.hidden_size // num_encoder_attention_heads,
     )

     _A          =      common_inputs['''attention_mask'''].dtype
     _A          =      torch.cat(
         [common_inputs['''attention_mask'''], torch.ones(a		,      a		,      dtype=a   )]		,      dim=1   )
     _A          =      [
         (torch.zeros(a   ), torch.zeros(a   )) for _ in range(a   )
     ]
   return common_inputs
 def lowercase_						(  self		,      a		,      a = -1		,      a = -1		,      a = False		,      a = None		,      )		->							Mapping[str, Any]:




   """simple docstring"""







   _A          =      compute_effective_axis_dimension(
       a		,      fixed_dimension=OnnxConfig.default_fixed_batch		,      num_token_to_add=0   )

   # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
   _A          =      tokenizer.num_special_tokens_to_add(a   )
   _A          =      compute_effective_axis_dimension(
       a		,      fixed_dimension=OnnxConfig.default_fixed_sequence		,      num_token_to_add=a   )

   # Generate dummy inputs according to compute batch and sequence
   _A          =      [''' '''.join([tokenizer.unk_token]   ) * seq_length] * batch_size
   _A          =      dict(tokenizer(a		,      return_tensors=a   )   )
   return common_inputs
 def lowercase_						(  self		,      a		,      a = -1		,      a = -1		,      a = False		,      a = None		,      )		->							Mapping[str, Any]:




   """simple docstring"""







   if self.task in ["default", "seq2seq-lm"]:
     _A          =      self._generate_dummy_inputs_for_default_and_seqaseq_lm(
         a		,      batch_size=a		,      seq_length=a		,      is_pair=a		,      framework=a   )

   elif self.task == "causal-lm":
     _A          =      self._generate_dummy_inputs_for_causal_lm(
         a		,      batch_size=a		,      seq_length=a		,      is_pair=a		,      framework=a   )
   else:
     _A          =      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
         a		,      batch_size=a		,      seq_length=a		,      is_pair=a		,      framework=a   )

   return common_inputs



 def lowercase_						(  self		,      a		,      a		,      a		,      a   )		->							List[str]:




   """simple docstring"""







   if self.task in ["default", "seq2seq-lm"]:
     _A          =      super()._flatten_past_key_values_(a		,      a		,      a		,      a   )
   else:
     _A          =      super(a		,      self   )._flatten_past_key_values_(
         a		,      a		,      a		,      a   )




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


__lowerCamelCase								=   logging.get_logger(__name__)

__lowerCamelCase								=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowerCamelCase								=   {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

__lowerCamelCase								=   {
    """microsoft/deberta-base""": 5_1_2,
    """microsoft/deberta-large""": 5_1_2,
    """microsoft/deberta-xlarge""": 5_1_2,
    """microsoft/deberta-base-mnli""": 5_1_2,
    """microsoft/deberta-large-mnli""": 5_1_2,
    """microsoft/deberta-xlarge-mnli""": 5_1_2,
}

__lowerCamelCase								=   {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}



class 						_snake_case (     lowerCamelCase			):





 """simple docstring"""



 lowerCamelCase_       =     VOCAB_FILES_NAMES
 lowerCamelCase_       =     PRETRAINED_VOCAB_FILES_MAP
 lowerCamelCase_       =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCamelCase_       =     ['''input_ids''', '''attention_mask''', '''token_type_ids''']
 lowerCamelCase_       =     DebertaTokenizer
 def __init__(  self		,      a=None		,      a=None		,      a=None		,      a="replace"		,      a="[CLS]"		,      a="[SEP]"		,      a="[SEP]"		,      a="[CLS]"		,      a="[UNK]"		,      a="[PAD]"		,      a="[MASK]"		,      a=False		,      **a		,      )		->							Any:




   """simple docstring"""







   super().__init__(
       a		,      a		,      tokenizer_file=a		,      errors=a		,      bos_token=a		,      eos_token=a		,      unk_token=a		,      sep_token=a		,      cls_token=a		,      pad_token=a		,      mask_token=a		,      add_prefix_space=a		,      **a		,      )
   _A          =      kwargs.pop('''add_bos_token'''		,      a   )

   _A          =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
   if pre_tok_state.get('''add_prefix_space'''		,      a   ) != add_prefix_space:
     _A          =      getattr(a		,      pre_tok_state.pop('''type'''   )   )
     _A          =      add_prefix_space
     _A          =      pre_tok_class(**a   )

   _A          =      add_prefix_space
 @property
 def lowercase_						(  self   )		->							str:




   """simple docstring"""







   if self._mask_token is None:
     if self.verbose:
       logger.error('''Using mask_token, but it is not set yet.'''   )
     return None
   return str(self._mask_token   )
 @mask_token.setter
 def lowercase_						(  self		,      a   )		->							Dict:




   """simple docstring"""







   _A          =      AddedToken(a		,      lstrip=a		,      rstrip=a   ) if isinstance(a		,      a   ) else value
   _A          =      value
 def lowercase_						(  self		,      a		,      a = None   )		->							List[int]:




   """simple docstring"""







   if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   _A          =      [self.cls_token_id]
   _A          =      [self.sep_token_id]
   return cls + token_ids_a + sep + token_ids_a + sep
 def lowercase_						(  self		,      a		,      a = None   )		->							List[int]:




   """simple docstring"""







   _A          =      [self.sep_token_id]
   _A          =      [self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep   ) * [0]
   return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]
 def lowercase_						(  self		,      *a		,      **a   )		->							BatchEncoding:




   """simple docstring"""







   _A          =      kwargs.get('''is_split_into_words'''		,      a   )
   assert self.add_prefix_space or not is_split_into_words, (
       f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*a		,      **a   )
 def lowercase_						(  self		,      *a		,      **a   )		->							BatchEncoding:




   """simple docstring"""







   _A          =      kwargs.get('''is_split_into_words'''		,      a   )

   assert self.add_prefix_space or not is_split_into_words, (
       f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*a		,      **a   )
 def lowercase_						(  self		,      a		,      a = None   )		->							Tuple[str]:




   """simple docstring"""







   _A          =      self._tokenizer.model.save(a		,      name=a   )
   return tuple(a   )



 def lowercase_						(  self		,      a   )		->							List[int]:




   """simple docstring"""







   _A          =      []
   for is_user, text in conversation.iter_texts():
     input_ids.extend(self.encode(a		,      add_special_tokens=a   ) + [self.eos_token_id]   )

   if len(a   ) > self.model_max_length:
     _A          =      input_ids[-self.model_max_length :]
   return input_ids