

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a		  =       logging.get_logger(__name__)

_a		  =       {
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}






class 							A_			(		_lowerCAmelCase ):
     _lowercase      :					List[Any]       =     "mega"


     def __init__(							self      :		int      ,  UpperCAmelCase      :		Dict=3_0_5_2_2      ,  UpperCAmelCase      :		List[str]=1_2_8      ,  UpperCAmelCase      :		int=4      ,  UpperCAmelCase      :		str=2_5_6      ,  UpperCAmelCase      :		List[Any]=1_6      ,  UpperCAmelCase      :		List[str]=True      ,  UpperCAmelCase      :		Any=6_4      ,  UpperCAmelCase      :		Optional[Any]=False      ,  UpperCAmelCase      :		Tuple=-1      ,  UpperCAmelCase      :		Any=None      ,  UpperCAmelCase      :		int=True      ,  UpperCAmelCase      :		Optional[Any]="scalenorm"      ,  UpperCAmelCase      :		Any=True      ,  UpperCAmelCase      :		Union[str, Any]="silu"      ,  UpperCAmelCase      :		Optional[Any]="softmax"      ,  UpperCAmelCase      :		Any=0.1      ,  UpperCAmelCase      :		Any=0.1      ,  UpperCAmelCase      :		Dict=0.1      ,  UpperCAmelCase      :		List[str]=False      ,  UpperCAmelCase      :		List[str]=True      ,  UpperCAmelCase      :		str=2_5_6      ,  UpperCAmelCase      :		Optional[Any]=True      ,  UpperCAmelCase      :		Optional[Any]=0.1      ,  UpperCAmelCase      :		int=2_0_4_8      ,  UpperCAmelCase      :		Any=False      ,  UpperCAmelCase      :		int=2      ,  UpperCAmelCase      :		Optional[int]=0.02      ,  UpperCAmelCase      :		str=0.2      ,  UpperCAmelCase      :		int=0.02      ,  UpperCAmelCase      :		Union[str, Any]=1.0      ,  UpperCAmelCase      :		Any=1      ,  UpperCAmelCase      :		List[Any]=0      ,  UpperCAmelCase      :		Optional[int]=2      ,  UpperCAmelCase      :		int="rotary"      ,  UpperCAmelCase      :		Any=None      ,  UpperCAmelCase      :		List[str]=True      ,  UpperCAmelCase      :		str=True      ,  **UpperCAmelCase      :		Optional[int]      ,  )							->     str:
      super().__init__(pad_token_id=_lowerCamelCase      ,  bos_token_id=_lowerCamelCase      ,  eos_token_id=_lowerCamelCase      ,  **_lowerCamelCase )

      __lowerCAmelCase:       Any            =				vocab_size
      __lowerCAmelCase:       Union[str, Any]            =				hidden_size
      __lowerCAmelCase:       Optional[Any]            =				num_hidden_layers
      __lowerCAmelCase:       List[str]            =				activation
      __lowerCAmelCase:       Union[str, Any]            =				attention_activation
      __lowerCAmelCase:       List[Any]            =				intermediate_size
      __lowerCAmelCase:       List[Any]            =				ema_projection_size
      __lowerCAmelCase:       Union[str, Any]            =				bidirectional
      __lowerCAmelCase:       Dict            =				shared_representation_size
      __lowerCAmelCase:       List[Any]            =				use_chunking
      __lowerCAmelCase:       Dict            =				chunk_size
      __lowerCAmelCase:       str            =				truncation
      __lowerCAmelCase:       Optional[int]            =				normalize_before_mega
      __lowerCAmelCase:       Any            =				normalization_type
      __lowerCAmelCase:       Any            =				norm_affine
      __lowerCAmelCase:       List[Any]            =				dropout_prob
      __lowerCAmelCase:       Dict            =				hidden_dropout_prob
      __lowerCAmelCase:       Dict            =				attention_probs_dropout_prob
      __lowerCAmelCase:       int            =				use_feature_dropout
      __lowerCAmelCase:       Optional[int]            =				use_normalized_ffn
      __lowerCAmelCase:       List[str]            =				nffn_hidden_size
      __lowerCAmelCase:       Dict            =				normalize_before_ffn
      __lowerCAmelCase:       str            =				nffn_activation_dropout_prob
      __lowerCAmelCase:       Dict            =				max_positions
      __lowerCAmelCase:       str            =				add_token_type_embeddings
      __lowerCAmelCase:       List[Any]            =				type_vocab_size
      __lowerCAmelCase:       str            =				initializer_range
      __lowerCAmelCase:       Optional[int]            =				ema_delta_alpha_range
      __lowerCAmelCase:       int            =				ema_beta_range
      __lowerCAmelCase:       int            =				ema_gamma_omega_range
      __lowerCAmelCase:       Union[str, Any]            =				relative_positional_bias
      __lowerCAmelCase:       Dict            =				use_cache
      __lowerCAmelCase:       int            =				classifier_dropout
      __lowerCAmelCase:       Optional[Any]            =				add_lm_hidden_dense_layer
      __lowerCAmelCase:       Any            =				1  # not used but required by Hugging Face







class 							A_			(		_lowerCAmelCase ):


     @property
     def   UpperCAmelCase  (							self      :		Union[str, Any] )							->     Tuple:
      if self.task == "multiple-choice":
       __lowerCAmelCase:       Any            =				{0: 'batch', 1: 'choice', 2: 'sequence'}
      else:
       __lowerCAmelCase:       List[str]            =				{0: 'batch', 1: 'sequence'}
      return OrderedDict(
          [
              ('input_ids', dynamic_axis),
              ('attention_mask', dynamic_axis),
          ] )








import torch

from transformers import PreTrainedModel

from .custom_configuration import CustomConfig, NoSuperInitConfig






class 							A_			(		snake_case__ ):
     _lowercase      :					Any       =     CustomConfig


     def __init__(							self      :		List[Any]      ,  UpperCAmelCase      :		Optional[Any] )							->     int:
      super().__init__(UpperCAmelCase )
      __lowerCAmelCase:       Tuple            =				torch.nn.Linear(config.hidden_size      ,  config.hidden_size )


     def   UpperCAmelCase  (							self      :		Optional[int]      ,  UpperCAmelCase      :		Tuple )							->     Union[str, Any]:
      return self.linear(UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		str      ,  UpperCAmelCase      :		str )							->     Optional[int]:
      pass







class 							A_			(		snake_case__ ):
     _lowercase      :					str       =     NoSuperInitConfig


     def __init__(							self      :		List[Any]      ,  UpperCAmelCase      :		Union[str, Any] )							->     int:
      super().__init__(UpperCAmelCase )
      __lowerCAmelCase:       int            =				torch.nn.Linear(config.attribute      ,  config.attribute )


     def   UpperCAmelCase  (							self      :		Tuple      ,  UpperCAmelCase      :		Tuple )							->     Tuple:
      return self.linear(UpperCAmelCase )


     def   UpperCAmelCase  (							self      :		Optional[Any]      ,  UpperCAmelCase      :		Any )							->     str:
      pass





