






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase  							= logging.get_logger(__name__)

lowercase  							= {
    "facebook/hubert-base-ls960": "https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json",
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}




class 			UpperCamelCase_				(      snake_case_    ):



	'''simple docstring'''




	lowerCAmelCase													=			'''hubert'''




	def __init__(	self	,      a=32	,      a=7_68	,      a=12	,      a=12	,      a=30_72	,      a="gelu"	,      a=0.1	,      a=0.1	,      a=0.1	,      a=True	,      a=0.0	,      a=0.1	,      a=0.1	,      a=0.02	,      a=1E-5	,      a="group"	,      a="gelu"	,      a=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)	,      a=(5, 2, 2, 2, 2, 2, 2)	,      a=(10, 3, 3, 3, 3, 2, 2)	,      a=False	,      a=1_28	,      a=16	,      a=False	,      a=True	,      a=0.05	,      a=10	,      a=2	,      a=0.0	,      a=10	,      a=0	,      a="sum"	,      a=False	,      a=False	,      a=2_56	,      a=0	,      a=1	,      a=2	,      **a	,      )	->   Dict:
								super().__init__(**a	,      pad_token_id=a	,      bos_token_id=a	,      eos_token_id=a )
								snake_case_           =							hidden_size
								snake_case_           =							feat_extract_norm
								snake_case_           =							feat_extract_activation
								snake_case_           =							list(a )
								snake_case_           =							list(a )
								snake_case_           =							list(a )
								snake_case_           =							conv_bias
								snake_case_           =							num_conv_pos_embeddings
								snake_case_           =							num_conv_pos_embedding_groups
								snake_case_           =							len(self.conv_dim )
								snake_case_           =							num_hidden_layers
								snake_case_           =							intermediate_size
								snake_case_           =							hidden_act
								snake_case_           =							num_attention_heads
								snake_case_           =							hidden_dropout
								snake_case_           =							attention_dropout
								snake_case_           =							activation_dropout
								snake_case_           =							feat_proj_layer_norm
								snake_case_           =							feat_proj_dropout
								snake_case_           =							final_dropout
								snake_case_           =							layerdrop
								snake_case_           =							layer_norm_eps
								snake_case_           =							initializer_range
								snake_case_           =							vocab_size
								snake_case_           =							do_stable_layer_norm
								snake_case_           =							use_weighted_layer_sum
								snake_case_           =							classifier_proj_size

								if (
								    (len(self.conv_stride ) != self.num_feat_extract_layers)
								    or (len(self.conv_kernel ) != self.num_feat_extract_layers)
								    or (len(self.conv_dim ) != self.num_feat_extract_layers)
								):
															raise ValueError(
															    'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
															    ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
															    F''' {len(self.conv_dim )}`, `len(config.conv_stride) = {len(self.conv_stride )}`,'''
															    F''' `len(config.conv_kernel) = {len(self.conv_kernel )}`.''' )

								# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
								snake_case_           =							apply_spec_augment
								snake_case_           =							mask_time_prob
								snake_case_           =							mask_time_length
								snake_case_           =							mask_time_min_masks
								snake_case_           =							mask_feature_prob
								snake_case_           =							mask_feature_length
								snake_case_           =							mask_feature_min_masks

								# ctc loss
								snake_case_           =							ctc_loss_reduction
								snake_case_           =							ctc_zero_infinity






	@property
	def 							_UpperCamelCase      (	self )	->   List[str]:
								return functools.reduce(operator.mul	,      self.conv_stride	,      1 )













import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
	import transformer_engine.pytorch as te





def 						__UpperCAmelCase      (					a_							,					a_=True							,					a_=True							,					a_=True):
							if not is_fpa_available():
														raise ImportError('Using `convert_model` requires transformer_engine to be installed.')
							for name, module in model.named_children():
														if isinstance(a_							,					nn.Linear) and to_transformer_engine and _convert_linear:
																					# Return early if the linear layer weights are not multiples of 16
																					if any(p % 16 != 0 for p in module.weight.shape):
																												return
																					snake_case_           =							module.bias is not None
																					snake_case_           =							te.Linear(
																					    module.in_features							,					module.out_features							,					bias=a_							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					if has_bias:
																												snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														elif isinstance(a_							,					nn.LayerNorm) and to_transformer_engine and _convert_ln:
																					snake_case_           =							te.LayerNorm(module.normalized_shape[0]							,					eps=module.eps							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														elif isinstance(a_							,					te.Linear) and not to_transformer_engine and _convert_linear:
																					snake_case_           =							module.bias is not None
																					snake_case_           =							nn.Linear(
																					    module.in_features							,					module.out_features							,					bias=a_							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					if has_bias:
																												snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														elif isinstance(a_							,					te.LayerNorm) and not to_transformer_engine and _convert_ln:
																					snake_case_           =							nn.LayerNorm(module.normalized_shape[0]							,					eps=module.eps							,					params_dtype=module.weight.dtype)
																					snake_case_           =							module.weight.data.clone()
																					snake_case_           =							module.bias.data.clone()

																					setattr(a_							,					a_							,					a_)
														else:
																					convert_model(
																					    a_							,					to_transformer_engine=a_							,					_convert_linear=_convert_linear							,					_convert_ln=_convert_ln							,					)





def 						__UpperCAmelCase      (					a_):
							if not is_fpa_available():
														raise ImportError('Using `has_transformer_engine_layers` requires transformer_engine to be installed.')
							for m in model.modules():
														if isinstance(a_							,					(te.LayerNorm, te.Linear)):
																					return True
							return False





