



'''simple docstring'''






import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
						import transformer_engine.pytorch as te


def 			snake_case_					( _lowerCAmelCase  :						Any       ,				_lowerCAmelCase  :						List[str]=True       ,				_lowerCAmelCase  :						Tuple=True       ,				_lowerCAmelCase  :						Dict=True  )				->       Tuple:
							if not is_fpa_available():
														raise ImportError('''Using `convert_model` requires transformer_engine to be installed.'''  )
							for name, module in model.named_children():
														if isinstance(_lowerCAmelCase       ,				nn.Linear  ) and to_transformer_engine and _convert_linear:
																					# Return early if the linear layer weights are not multiples of 16
																					if any(p % 16 != 0 for p in module.weight.shape  ):
																												return
																					UpperCAmelCase			:	int										=		module.bias is not None
																					UpperCAmelCase			:	int										=		te.Linear(
																					    module.in_features       ,				module.out_features       ,				bias=_lowerCAmelCase       ,				params_dtype=module.weight.dtype  )
																					UpperCAmelCase			:	Optional[Any]										=		module.weight.data.clone()
																					if has_bias:
																												UpperCAmelCase			:	Union[str, Any]										=		module.bias.data.clone()

																					setattr(_lowerCAmelCase       ,				_lowerCAmelCase       ,				_lowerCAmelCase  )
														elif isinstance(_lowerCAmelCase       ,				nn.LayerNorm  ) and to_transformer_engine and _convert_ln:
																					UpperCAmelCase			:	Optional[int]										=		te.LayerNorm(module.normalized_shape[0]       ,				eps=module.eps       ,				params_dtype=module.weight.dtype  )
																					UpperCAmelCase			:	Any										=		module.weight.data.clone()
																					UpperCAmelCase			:	Optional[int]										=		module.bias.data.clone()

																					setattr(_lowerCAmelCase       ,				_lowerCAmelCase       ,				_lowerCAmelCase  )
														elif isinstance(_lowerCAmelCase       ,				te.Linear  ) and not to_transformer_engine and _convert_linear:
																					UpperCAmelCase			:	Dict										=		module.bias is not None
																					UpperCAmelCase			:	Optional[int]										=		nn.Linear(
																					    module.in_features       ,				module.out_features       ,				bias=_lowerCAmelCase       ,				params_dtype=module.weight.dtype  )
																					UpperCAmelCase			:	int										=		module.weight.data.clone()
																					if has_bias:
																												UpperCAmelCase			:	str										=		module.bias.data.clone()

																					setattr(_lowerCAmelCase       ,				_lowerCAmelCase       ,				_lowerCAmelCase  )
														elif isinstance(_lowerCAmelCase       ,				te.LayerNorm  ) and not to_transformer_engine and _convert_ln:
																					UpperCAmelCase			:	Tuple										=		nn.LayerNorm(module.normalized_shape[0]       ,				eps=module.eps       ,				params_dtype=module.weight.dtype  )
																					UpperCAmelCase			:	Optional[int]										=		module.weight.data.clone()
																					UpperCAmelCase			:	Dict										=		module.bias.data.clone()

																					setattr(_lowerCAmelCase       ,				_lowerCAmelCase       ,				_lowerCAmelCase  )
														else:
																					convert_model(
																					    _lowerCAmelCase       ,				to_transformer_engine=_lowerCAmelCase       ,				_convert_linear=_convert_linear       ,				_convert_ln=_convert_ln       ,				)




def 			snake_case_					( _lowerCAmelCase  :						Optional[Any]  )				->       Any:
							if not is_fpa_available():
														raise ImportError('''Using `has_transformer_engine_layers` requires transformer_engine to be installed.'''  )
							for m in model.modules():
														if isinstance(_lowerCAmelCase       ,				(te.LayerNorm, te.Linear)  ):
																					return True
							return False










'''simple docstring'''






UpperCamelCase__:       str							     =      [0.485, 0.456, 0.406]
UpperCamelCase__:       List[str]							     =      [0.229, 0.224, 0.225]
UpperCamelCase__:       List[str]							     =      [0.5, 0.5, 0.5]
UpperCamelCase__:       Optional[Any]							     =      [0.5, 0.5, 0.5]
UpperCamelCase__:       List[str]							     =      [0.48145466, 0.4578275, 0.40821073]
UpperCamelCase__:       List[str]							     =      [0.26862954, 0.26130258, 0.27577711]





