






'''simple docstring'''
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


UpperCamelCase__	=     logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

UpperCamelCase__	=     {
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
UpperCamelCase__	=     {
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
UpperCamelCase__	=     sorted(arg_to_scheduler.keys())
UpperCamelCase__	=     '{' + ', '.join(arg_to_scheduler_choices) + '}'


class     _UpperCAmelCase  (						pl.LightningModule     ):




    def __init__(     self					:    Union[str, Any]					,       a					:    argparse.Namespace					,       a					:    Union[str, Any]=None					,       a					:    List[Any]="base"					,       a					:    Optional[int]=None					,       a					:    Optional[Any]=None					,       a					:    Any=None					,       **a					:    int					,       ):






       '''simple docstring'''






       super().__init__()
       # TODO: move to self.save_hyperparameters()
       # self.save_hyperparameters()
       # can also expand arguments into trainer signature for easier reading

       self.save_hyperparameters(a     )
       lowercase_		:      Optional[Any]          =       0
       lowercase_		:      Dict          =       Path(self.hparams.output_dir     )
       lowercase_		:      Dict          =       self.hparams.cache_dir if self.hparams.cache_dir else None
       if config is None:
          lowercase_		:      str          =       AutoConfig.from_pretrained(
              self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path					,       **({"num_labels": num_labels} if num_labels is not None else {})					,       cache_dir=a					,       **a					,       )
       else:
          lowercase_		:      PretrainedConfig          =       config

       lowercase_		:      Optional[Any]          =       ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
       for p in extra_model_params:
          if getattr(self.hparams					,       a					,       a     ):
             assert hasattr(self.config					,       a     ), f"""model config doesn't have a `{p}` attribute"""
             setattr(self.config					,       a					,       getattr(self.hparams					,       a     )     )

       if tokenizer is None:
          lowercase_		:      Optional[Any]          =       AutoTokenizer.from_pretrained(
              self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path					,       cache_dir=a					,       )
       else:
          lowercase_		:      PreTrainedTokenizer          =       tokenizer
       lowercase_		:      Union[str, Any]          =       MODEL_MODES[mode]
       if model is None:
          lowercase_		:      Union[str, Any]          =       self.model_type.from_pretrained(
              self.hparams.model_name_or_path					,       from_tf=bool(".ckpt" in self.hparams.model_name_or_path     )					,       config=self.config					,       cache_dir=a					,       )
       else:
          lowercase_		:      Union[str, Any]          =       model




    def 			lowerCAmelCase__			(     self					:    List[Any]					,       *a					:    List[str]					,       **a					:    Dict     ):






       '''simple docstring'''






       lowercase_		:      str          =       self.model_type.from_pretrained(*a					,       **a     )




    def 			lowerCAmelCase__			(     self					:    Union[str, Any]     ):






       '''simple docstring'''






       lowercase_		:      str          =       arg_to_scheduler[self.hparams.lr_scheduler]
       lowercase_		:      Dict          =       get_schedule_func(
           self.opt					,       num_warmup_steps=self.hparams.warmup_steps					,       num_training_steps=self.total_steps()     )
       lowercase_		:      str          =       {"scheduler": scheduler, "interval": "step", "frequency": 1}
       return scheduler




    def 			lowerCAmelCase__			(     self					:    Union[str, Any]     ):






       '''simple docstring'''






       lowercase_		:      List[str]          =       self.model
       lowercase_		:      Union[str, Any]          =       ["bias", "LayerNorm.weight"]
       lowercase_		:      Optional[int]          =       [
           {
               "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay     )],
               "weight_decay": self.hparams.weight_decay,
           },
           {
               "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay     )],
               "weight_decay": 0.0,
           },
       ]
       if self.hparams.adafactor:
          lowercase_		:      int          =       Adafactor(
              a					,       lr=self.hparams.learning_rate					,       scale_parameter=a					,       relative_step=a     )

       else:
          lowercase_		:      List[Any]          =       AdamW(
              a					,       lr=self.hparams.learning_rate					,       eps=self.hparams.adam_epsilon     )
       lowercase_		:      Any          =       optimizer

       lowercase_		:      Union[str, Any]          =       self.get_lr_scheduler()

       return [optimizer], [scheduler]




    def 			lowerCAmelCase__			(     self					:    Optional[Any]					,       a					:    int					,       a					:    List[Any]     ):






       '''simple docstring'''






       return self.validation_step(a					,       a     )




    def 			lowerCAmelCase__			(     self					:    int					,       a					:    List[str]     ):






       '''simple docstring'''






       return self.validation_end(a     )




    def 			lowerCAmelCase__			(     self					:    str     ):






       '''simple docstring'''






       lowercase_		:      Dict          =       max(1					,       self.hparams.gpus     )  # TODO: consider num_tpu_cores
       lowercase_		:      Union[str, Any]          =       self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
       return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs




    def 			lowerCAmelCase__			(     self					:    List[Any]					,       a					:    Any     ):






       '''simple docstring'''






       if mode == "test":
          lowercase_		:      List[str]          =       len(self.test_dataloader().dataset     )
       else:
          lowercase_		:      int          =       self.get_dataloader("train"					,       self.hparams.train_batch_size					,       shuffle=a     )
          lowercase_		:      Union[str, Any]          =       len(self.train_dataloader().dataset     )




    def 			lowerCAmelCase__			(     self					:    List[Any]					,       a					:    str					,       a					:    int					,       a					:    bool = False     ):






       '''simple docstring'''






       raise NotImplementedError("You must implement this for your task"     )




    def 			lowerCAmelCase__			(     self					:    Any     ):






       '''simple docstring'''






       return self.train_loader




    def 			lowerCAmelCase__			(     self					:    str     ):






       '''simple docstring'''






       return self.get_dataloader("dev"					,       self.hparams.eval_batch_size					,       shuffle=a     )




    def 			lowerCAmelCase__			(     self					:    Tuple     ):






       '''simple docstring'''






       return self.get_dataloader("test"					,       self.hparams.eval_batch_size					,       shuffle=a     )




    def 			lowerCAmelCase__			(     self					:    int					,       a					:    Any     ):






       '''simple docstring'''






       return os.path.join(
           self.hparams.data_dir					,       "cached_{}_{}_{}".format(
               a					,       list(filter(a					,       self.hparams.model_name_or_path.split("/"     )     )     ).pop()					,       str(self.hparams.max_seq_length     )					,       )					,       )




    @pl.utilities.rank_zero_only
    def 			lowerCAmelCase__			(     self					:    Optional[int]					,       a					:    Dict[str, Any]     ):






       '''simple docstring'''






       lowercase_		:      List[str]          =       self.output_dir.joinpath("best_tfmr"     )
       lowercase_		:      Tuple          =       self.step_count
       self.model.save_pretrained(a     )
       self.tokenizer.save_pretrained(a     )





    @staticmethod
    def 			lowerCAmelCase__			(     a					:    List[str]					,       a					:    int     ):






       '''simple docstring'''






       parser.add_argument(
           "--model_name_or_path"					,       default=a					,       type=a					,       required=a					,       help="Path to pretrained model or model identifier from huggingface.co/models"					,       )
       parser.add_argument(
           "--config_name"					,       default=""					,       type=a					,       help="Pretrained config name or path if not the same as model_name"     )
       parser.add_argument(
           "--tokenizer_name"					,       default=a					,       type=a					,       help="Pretrained tokenizer name or path if not the same as model_name"					,       )
       parser.add_argument(
           "--cache_dir"					,       default=""					,       type=a					,       help="Where do you want to store the pre-trained models downloaded from huggingface.co"					,       )
       parser.add_argument(
           "--encoder_layerdrop"					,       type=a					,       help="Encoder layer dropout probability (Optional). Goes into model.config"					,       )
       parser.add_argument(
           "--decoder_layerdrop"					,       type=a					,       help="Decoder layer dropout probability (Optional). Goes into model.config"					,       )
       parser.add_argument(
           "--dropout"					,       type=a					,       help="Dropout probability (Optional). Goes into model.config"					,       )
       parser.add_argument(
           "--attention_dropout"					,       type=a					,       help="Attention dropout probability (Optional). Goes into model.config"					,       )
       parser.add_argument("--learning_rate"					,       default=5e-5					,       type=a					,       help="The initial learning rate for Adam."     )
       parser.add_argument(
           "--lr_scheduler"					,       default="linear"					,       choices=a					,       metavar=a					,       type=a					,       help="Learning rate scheduler"					,       )
       parser.add_argument("--weight_decay"					,       default=0.0					,       type=a					,       help="Weight decay if we apply some."     )
       parser.add_argument("--adam_epsilon"					,       default=1e-8					,       type=a					,       help="Epsilon for Adam optimizer."     )
       parser.add_argument("--warmup_steps"					,       default=0					,       type=a					,       help="Linear warmup over warmup_steps."     )
       parser.add_argument("--num_workers"					,       default=4					,       type=a					,       help="kwarg passed to DataLoader"     )
       parser.add_argument("--num_train_epochs"					,       dest="max_epochs"					,       default=3					,       type=a     )
       parser.add_argument("--train_batch_size"					,       default=3_2					,       type=a     )
       parser.add_argument("--eval_batch_size"					,       default=3_2					,       type=a     )
       parser.add_argument("--adafactor"					,       action="store_true"     )


class     _UpperCAmelCase  (						pl.Callback     ):




    def 			lowerCAmelCase__			(     self					:    Union[str, Any]					,       a					:    int					,       a					:    Optional[Any]     ):






       '''simple docstring'''






       lowercase_		:      int          =       trainer.lr_schedulers[0]["scheduler"]
       lowercase_		:      str          =       {f"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()     )}
       pl_module.logger.log_metrics(a     )




    def 			lowerCAmelCase__			(     self					:    Tuple					,       a					:    pl.Trainer					,       a					:    pl.LightningModule     ):






       '''simple docstring'''






       rank_zero_info("***** Validation results *****"     )
       lowercase_		:      Dict          =       trainer.callback_metrics
       # Log results
       for key in sorted(a     ):
          if key not in ["log", "progress_bar"]:
             rank_zero_info("{} = {}\n".format(a					,       str(metrics[key]     )     )     )





    def 			lowerCAmelCase__			(     self					:    Any					,       a					:    pl.Trainer					,       a					:    pl.LightningModule     ):






       '''simple docstring'''






       rank_zero_info("***** Test results *****"     )
       lowercase_		:      Any          =       trainer.callback_metrics
       # Log and save results to file
       lowercase_		:      Tuple          =       os.path.join(pl_module.hparams.output_dir					,       "test_results.txt"     )
       with open(a					,       "w"     ) as writer:
          for key in sorted(a     ):
             if key not in ["log", "progress_bar"]:
                rank_zero_info("{} = {}\n".format(a					,       str(metrics[key]     )     )     )
                writer.write("{} = {}\n".format(a					,       str(metrics[key]     )     )     )




def 						__SCREAMING_SNAKE_CASE   (					_UpperCamelCase  ,						_UpperCamelCase							):


   """simple docstring"""




   parser.add_argument(
       "--output_dir"  ,						default=_UpperCamelCase  ,						type=_UpperCamelCase  ,						required=_UpperCamelCase  ,						help="The output directory where the model predictions and checkpoints will be written."  ,						)
   parser.add_argument(
       "--fp16"  ,						action="store_true"  ,						help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"  ,						)

   parser.add_argument(
       "--fp16_opt_level"  ,						type=_UpperCamelCase  ,						default="O2"  ,						help=(
           "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
           "See details at https://nvidia.github.io/apex/amp.html"
       )  ,						)
   parser.add_argument("--n_tpu_cores"  ,						dest="tpu_cores"  ,						type=_UpperCamelCase							)
   parser.add_argument("--max_grad_norm"  ,						dest="gradient_clip_val"  ,						default=1.0  ,						type=_UpperCamelCase  ,						help="Max gradient norm"							)
   parser.add_argument("--do_train"  ,						action="store_true"  ,						help="Whether to run training."							)
   parser.add_argument("--do_predict"  ,						action="store_true"  ,						help="Whether to run predictions on the test set."							)
   parser.add_argument(
       "--gradient_accumulation_steps"  ,						dest="accumulate_grad_batches"  ,						type=_UpperCamelCase  ,						default=1  ,						help="Number of updates steps to accumulate before performing a backward/update pass."  ,						)
   parser.add_argument("--seed"  ,						type=_UpperCamelCase  ,						default=42  ,						help="random seed for initialization"							)
   parser.add_argument(
       "--data_dir"  ,						default=_UpperCamelCase  ,						type=_UpperCamelCase  ,						required=_UpperCamelCase  ,						help="The input data dir. Should contain the training files for the CoNLL-2003 NER task."  ,						)





def 						__SCREAMING_SNAKE_CASE   (					_UpperCamelCase  ,						_UpperCamelCase  ,						_UpperCamelCase=None  ,						_UpperCamelCase=True  ,						_UpperCamelCase=[]  ,						_UpperCamelCase=None  ,						_UpperCamelCase=None  ,						**_UpperCamelCase  ,						):


   """simple docstring"""




   pl.seed_everything(args.seed							)

   # init model
   lowercase_		:      Optional[int]          =       Path(model.hparams.output_dir							)
   odir.mkdir(exist_ok=_UpperCamelCase							)

   # add custom checkpoints
   if checkpoint_callback is None:
      lowercase_		:      Tuple          =       pl.callbacks.ModelCheckpoint(
          filepath=args.output_dir  ,						prefix="checkpoint"  ,						monitor="val_loss"  ,						mode="min"  ,						save_top_k=1							)
   if early_stopping_callback:
      extra_callbacks.append(_UpperCamelCase							)
   if logging_callback is None:
      lowercase_		:      List[Any]          =       LoggingCallback()

   lowercase_		:      Union[str, Any]          =       {}

   # TODO: remove with PyTorch 1.6 since pl uses native amp
   if args.fpaa:
      lowercase_		:      int          =       16
      lowercase_		:      str          =       args.fpaa_opt_level

   if args.gpus > 1:
      lowercase_		:      int          =       "ddp"

   lowercase_		:      List[str]          =       args.accumulate_grad_batches
   lowercase_		:      List[str]          =       extra_train_kwargs.get("accelerator"  ,						_UpperCamelCase							)
   lowercase_		:      Optional[int]          =       extra_train_kwargs.get("profiler"  ,						_UpperCamelCase							)

   lowercase_		:      List[Any]          =       pl.Trainer.from_argparse_args(
       _UpperCamelCase  ,						weights_summary=_UpperCamelCase  ,						callbacks=[logging_callback] + extra_callbacks  ,						logger=_UpperCamelCase  ,						checkpoint_callback=_UpperCamelCase  ,						**_UpperCamelCase  ,						)

   if args.do_train:
      trainer.fit(_UpperCamelCase							)

   return trainer












'''simple docstring'''
from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


UpperCamelCase__	=     logging.get_logger(__name__)

UpperCamelCase__	=     {
    'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}


class     _UpperCAmelCase  (						snake_case     ):
    __lowerCamelCase:				List[Any]						    =      'm2m_100'
    __lowerCamelCase:				Union[str, Any]						    =      ['past_key_values']
    __lowerCamelCase:				Union[str, Any]						    =      {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}




    def __init__(     self					:    str					,       a					:    Optional[Any]=1_2_8_1_1_2					,       a					:    Optional[int]=1_0_2_4					,       a					:    Any=1_2					,       a					:    Tuple=4_0_9_6					,       a					:    Any=1_6					,       a					:    Any=1_2					,       a					:    Dict=4_0_9_6					,       a					:    List[str]=1_6					,       a					:    List[Any]=0.05					,       a					:    Optional[int]=0.05					,       a					:    List[Any]=True					,       a					:    List[Any]=True					,       a					:    Union[str, Any]="relu"					,       a					:    Optional[Any]=1_0_2_4					,       a					:    List[str]=0.1					,       a					:    Union[str, Any]=0.1					,       a					:    int=0.0					,       a					:    List[str]=0.02					,       a					:    Optional[int]=2					,       a					:    Tuple=True					,       a					:    List[str]=1					,       a					:    int=0					,       a					:    int=2					,       **a					:    Tuple					,       ):






       '''simple docstring'''






       lowercase_		:      List[str]          =       vocab_size
       lowercase_		:      Tuple          =       max_position_embeddings
       lowercase_		:      int          =       d_model
       lowercase_		:      Optional[int]          =       encoder_ffn_dim
       lowercase_		:      Any          =       encoder_layers
       lowercase_		:      Optional[Any]          =       encoder_attention_heads
       lowercase_		:      List[Any]          =       decoder_ffn_dim
       lowercase_		:      Union[str, Any]          =       decoder_layers
       lowercase_		:      Optional[Any]          =       decoder_attention_heads
       lowercase_		:      Optional[int]          =       dropout
       lowercase_		:      Optional[Any]          =       attention_dropout
       lowercase_		:      Optional[int]          =       activation_dropout
       lowercase_		:      Any          =       activation_function
       lowercase_		:      Optional[Any]          =       init_std
       lowercase_		:      Tuple          =       encoder_layerdrop
       lowercase_		:      Optional[Any]          =       decoder_layerdrop
       lowercase_		:      List[Any]          =       use_cache
       lowercase_		:      Dict          =       encoder_layers
       lowercase_		:      Tuple          =       scale_embedding  # scale factor will be sqrt(d_model) if True

       super().__init__(
           pad_token_id=a					,       bos_token_id=a					,       eos_token_id=a					,       is_encoder_decoder=a					,       decoder_start_token_id=a					,       **a					,       )


class     _UpperCAmelCase  (						snake_case     ):




    @property
    def 			lowerCAmelCase__			(     self					:    Optional[Any]     ):






       '''simple docstring'''






       lowercase_		:      List[Any]          =       OrderedDict(
           [
               ("input_ids", {0: "batch", 1: "encoder_sequence"}),
               ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
           ]     )

       if self.use_past:
          lowercase_		:      Optional[int]          =       {0: "batch"}
          lowercase_		:      str          =       {0: "batch", 1: "past_decoder_sequence + sequence"}
       else:
          lowercase_		:      Optional[Any]          =       {0: "batch", 1: "decoder_sequence"}
          lowercase_		:      Union[str, Any]          =       {0: "batch", 1: "decoder_sequence"}

       if self.use_past:
          self.fill_with_past_key_values_(a					,       direction="inputs"     )
       return common_inputs




    def 			lowerCAmelCase__			(     self					:    Dict					,       a					:    PreTrainedTokenizer					,       a					:    int = -1					,       a					:    int = -1					,       a					:    bool = False					,       a					:    Optional[TensorType] = None					,       ):






       '''simple docstring'''






       lowercase_		:      int          =       compute_effective_axis_dimension(
           a					,       fixed_dimension=OnnxConfig.default_fixed_batch					,       num_token_to_add=0     )

       # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
       lowercase_		:      Dict          =       tokenizer.num_special_tokens_to_add(a     )
       lowercase_		:      str          =       compute_effective_axis_dimension(
           a					,       fixed_dimension=OnnxConfig.default_fixed_sequence					,       num_token_to_add=a     )

       # Generate dummy inputs according to compute batch and sequence
       lowercase_		:      Union[str, Any]          =       [" ".join([tokenizer.unk_token]     ) * seq_length] * batch_size
       lowercase_		:      Tuple          =       dict(tokenizer(a					,       return_tensors=a     )     )
       return common_inputs




    def 			lowerCAmelCase__			(     self					:    Any					,       a					:    PreTrainedTokenizer					,       a					:    int = -1					,       a					:    int = -1					,       a					:    bool = False					,       a					:    Optional[TensorType] = None					,       ):






       '''simple docstring'''






       lowercase_		:      Optional[Any]          =       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           a					,       a					,       a					,       a					,       a     )

       # Generate decoder inputs
       lowercase_		:      List[str]          =       seq_length if not self.use_past else 1
       lowercase_		:      Dict          =       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           a					,       a					,       a					,       a					,       a     )
       lowercase_		:      Union[str, Any]          =       {f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
       lowercase_		:      Union[str, Any]          =       dict(**a					,       **a     )

       if self.use_past:
          if not is_torch_available():
             raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."     )
          else:
             import torch
          lowercase_      ,						lowercase_		:      Union[str, Any]          =       common_inputs["input_ids"].shape
          lowercase_		:      List[Any]          =       common_inputs["decoder_input_ids"].shape[1]
          lowercase_      ,						lowercase_		:      Optional[Any]          =       self.num_attention_heads
          lowercase_		:      Any          =       (
              batch,
              num_encoder_attention_heads,
              encoder_seq_length,
              self._config.hidden_size // num_encoder_attention_heads,
          )
          lowercase_		:      Optional[Any]          =       decoder_seq_length + 3
          lowercase_		:      int          =       (
              batch,
              num_decoder_attention_heads,
              decoder_past_length,
              self._config.hidden_size // num_decoder_attention_heads,
          )

          lowercase_		:      Dict          =       torch.cat(
              [common_inputs["decoder_attention_mask"], torch.ones(a					,       a     )]					,       dim=1     )

          lowercase_		:      Tuple          =       []
          # If the number of encoder and decoder layers are present in the model configuration, both are considered
          lowercase_      ,						lowercase_		:      Tuple          =       self.num_layers
          lowercase_		:      Optional[Any]          =       min(a					,       a     )
          lowercase_		:      Tuple          =       max(a					,       a     ) - min_num_layers
          lowercase_		:      Optional[int]          =       "encoder" if num_encoder_layers > num_decoder_layers else "decoder"

          for _ in range(a     ):
             common_inputs["past_key_values"].append(
                 (
                     torch.zeros(a     ),
                     torch.zeros(a     ),
                     torch.zeros(a     ),
                     torch.zeros(a     ),
                 )     )
          # TODO: test this.
          lowercase_		:      Union[str, Any]          =       encoder_shape if remaining_side_name == "encoder" else decoder_shape
          for _ in range(a					,       a     ):
             common_inputs["past_key_values"].append((torch.zeros(a     ), torch.zeros(a     ))     )
       return common_inputs





    __lowerCamelCase:				str						    =      _generate_dummy_inputs_for_default_and_seqaseq_lm




