




'''simple docstring'''







import math

import torch
from torch import nn
from torch.nn import init

from .binarizer import MagnitudeBinarizer, ThresholdBinarizer, TopKBinarizer





class 					lowerCamelCase_							(	nn.Linear							):




					'''simple docstring'''






					def __init__(   self,  __UpperCamelCase,  __UpperCamelCase,  __UpperCamelCase = True,  __UpperCamelCase = "constant",  __UpperCamelCase = 0.0,  __UpperCamelCase = "topK",  ):
							super(__UpperCamelCase,  self			).__init__(in_features=__UpperCamelCase,  out_features=__UpperCamelCase,  bias=__UpperCamelCase			)
							assert pruning_method in ["topK", "threshold", "sigmoied_threshold", "magnitude", "l0"]
							_snake_case     	=	pruning_method

							if self.pruning_method in ["topK", "threshold", "sigmoied_threshold", "l0"]:
									_snake_case     	=	mask_scale
									_snake_case     	=	mask_init
									_snake_case     	=	nn.Parameter(torch.empty(self.weight.size()			)			)
									self.init_mask()






					def     __SCREAMING_SNAKE_CASE     (   self			):
							if self.mask_init == "constant":
									init.constant_(self.mask_scores,  val=self.mask_scale			)
							elif self.mask_init == "uniform":
									init.uniform_(self.mask_scores,  a=-self.mask_scale,  b=self.mask_scale			)
							elif self.mask_init == "kaiming":
									init.kaiming_uniform_(self.mask_scores,  a=math.sqrt(5			)			)






					def     __SCREAMING_SNAKE_CASE     (   self,  __UpperCamelCase,  __UpperCamelCase			):
							# Get the mask
							if self.pruning_method == "topK":
									_snake_case     	=	TopKBinarizer.apply(self.mask_scores,  __UpperCamelCase			)
							elif self.pruning_method in ["threshold", "sigmoied_threshold"]:
									_snake_case     	=	"sigmoied" in self.pruning_method
									_snake_case     	=	ThresholdBinarizer.apply(self.mask_scores,  __UpperCamelCase,  __UpperCamelCase			)
							elif self.pruning_method == "magnitude":
									_snake_case     	=	MagnitudeBinarizer.apply(self.weight,  __UpperCamelCase			)
							elif self.pruning_method == "l0":
									_snake_case	,   _snake_case	,   _snake_case     	=	-0.1, 1.1, 2 / 3
									if self.training:
											_snake_case     	=	torch.zeros_like(self.mask_scores			).uniform_().clamp(0.00_01,  0.99_99			)
											_snake_case     	=	torch.sigmoid((u.log() - (1 - u).log() + self.mask_scores) / b			)
									else:
											_snake_case     	=	torch.sigmoid(self.mask_scores			)
									_snake_case     	=	s * (r - l) + l
									_snake_case     	=	s_bar.clamp(min=0.0,  max=1.0			)
							# Mask weights with computed mask
							_snake_case     	=	mask * self.weight
							# Compute output (linear layer) with masked weights
							return nn.functional.linear(__UpperCamelCase,  __UpperCamelCase,  self.bias			)











'''simple docstring'''







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCamelCase							:						Optional[int]							     =  logging.get_logger(__name__)

_lowerCamelCase							:						int							     =  {
    'flaubert/flaubert_small_cased': 'https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/config.json',
    'flaubert/flaubert_base_uncased': 'https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/config.json',
    'flaubert/flaubert_base_cased': 'https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/config.json',
    'flaubert/flaubert_large_cased': 'https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/config.json',
}





class 					lowerCamelCase_							(	UpperCAmelCase_							):




					'''simple docstring'''






					lowercase__   =			"""flaubert"""
					lowercase__   =			{
					    """hidden_size""": """emb_dim""",
					    """num_attention_heads""": """n_heads""",
					    """num_hidden_layers""": """n_layers""",
					    """n_words""": """vocab_size""",  # For backward compatibility
					}






					def __init__(   self,  __UpperCamelCase=False,  __UpperCamelCase=0.0,  __UpperCamelCase=30_145,  __UpperCamelCase=2_048,  __UpperCamelCase=12,  __UpperCamelCase=16,  __UpperCamelCase=0.1,  __UpperCamelCase=0.1,  __UpperCamelCase=True,  __UpperCamelCase=False,  __UpperCamelCase=False,  __UpperCamelCase=False,  __UpperCamelCase=1,  __UpperCamelCase=True,  __UpperCamelCase=512,  __UpperCamelCase=2_048**-0.5,  __UpperCamelCase=1E-12,  __UpperCamelCase=0.02,  __UpperCamelCase=0,  __UpperCamelCase=1,  __UpperCamelCase=2,  __UpperCamelCase=3,  __UpperCamelCase=5,  __UpperCamelCase=True,  __UpperCamelCase="first",  __UpperCamelCase=True,  __UpperCamelCase=None,  __UpperCamelCase=True,  __UpperCamelCase=0.1,  __UpperCamelCase=5,  __UpperCamelCase=5,  __UpperCamelCase=0,  __UpperCamelCase=0,  __UpperCamelCase=2,  __UpperCamelCase=0,  **__UpperCamelCase,  ):
							_snake_case     	=	pre_norm
							_snake_case     	=	layerdrop
							_snake_case     	=	vocab_size
							_snake_case     	=	emb_dim
							_snake_case     	=	n_layers
							_snake_case     	=	n_heads
							_snake_case     	=	dropout
							_snake_case     	=	attention_dropout
							_snake_case     	=	gelu_activation
							_snake_case     	=	sinusoidal_embeddings
							_snake_case     	=	causal
							_snake_case     	=	asm
							_snake_case     	=	n_langs
							_snake_case     	=	use_lang_emb
							_snake_case     	=	layer_norm_eps
							_snake_case     	=	bos_index
							_snake_case     	=	eos_index
							_snake_case     	=	pad_index
							_snake_case     	=	unk_index
							_snake_case     	=	mask_index
							_snake_case     	=	is_encoder
							_snake_case     	=	max_position_embeddings
							_snake_case     	=	embed_init_std
							_snake_case     	=	init_std
							_snake_case     	=	summary_type
							_snake_case     	=	summary_use_proj
							_snake_case     	=	summary_activation
							_snake_case     	=	summary_proj_to_labels
							_snake_case     	=	summary_first_dropout
							_snake_case     	=	start_n_top
							_snake_case     	=	end_n_top
							_snake_case     	=	mask_token_id
							_snake_case     	=	lang_id

							if "n_words" in kwargs:
									_snake_case     	=	kwargs["n_words"]

							super().__init__(pad_token_id=__UpperCamelCase,  bos_token_id=__UpperCamelCase,  **__UpperCamelCase			)





class 					lowerCamelCase_							(	UpperCAmelCase_							):




					'''simple docstring'''






					@property
					def     __SCREAMING_SNAKE_CASE     (   self			):
							if self.task == "multiple-choice":
									_snake_case     	=	{0: "batch", 1: "choice", 2: "sequence"}
							else:
									_snake_case     	=	{0: "batch", 1: "sequence"}
							return OrderedDict(
							    [
							        ("input_ids", dynamic_axis),
							        ("attention_mask", dynamic_axis),
							    ]			)





