






import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase   :						str							=  logging.get_logger(__name__)

__UpperCamelCase   :						Optional[int]							=  {}


class 					__SCREAMING_SNAKE_CASE(		a_		):
							_UpperCAmelCase	      =    "fsmt_decoder"
							def __init__(       self: int    ,		UpperCamelCase: List[str]=0    ,		UpperCamelCase: Optional[int]=0  )						->						List[str]:
											super().__init__()
											snake_case__													=       vocab_size
											snake_case__													=       bos_token_id


class 					__SCREAMING_SNAKE_CASE(		a_		):
							_UpperCAmelCase	      =    "fsmt"
							_UpperCAmelCase	      =    {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}
							def __init__(       self: Dict    ,		UpperCamelCase: Tuple=["en", "de"]    ,		UpperCamelCase: str=4_20_24    ,		UpperCamelCase: Tuple=4_20_24    ,		UpperCamelCase: Optional[int]="relu"    ,		UpperCamelCase: Optional[Any]=10_24    ,		UpperCamelCase: Union[str, Any]=2_00    ,		UpperCamelCase: List[str]=10_24    ,		UpperCamelCase: List[Any]=40_96    ,		UpperCamelCase: List[Any]=12    ,		UpperCamelCase: Any=16    ,		UpperCamelCase: Any=0.0    ,		UpperCamelCase: Optional[Any]=40_96    ,		UpperCamelCase: str=12    ,		UpperCamelCase: int=16    ,		UpperCamelCase: Tuple=0.0    ,		UpperCamelCase: str=0.0    ,		UpperCamelCase: str=0.1    ,		UpperCamelCase: Any=0.0    ,		UpperCamelCase: List[Any]=0.02    ,		UpperCamelCase: List[Any]=2    ,		UpperCamelCase: Optional[Any]=True    ,		UpperCamelCase: Any=True    ,		UpperCamelCase: Optional[int]=False    ,		UpperCamelCase: Tuple=5    ,		UpperCamelCase: List[str]=1.0    ,		UpperCamelCase: Union[str, Any]=False    ,		UpperCamelCase: List[Any]=True    ,		UpperCamelCase: Union[str, Any]=1    ,		UpperCamelCase: Optional[Any]=0    ,		UpperCamelCase: Optional[Any]=2    ,		UpperCamelCase: List[Any]=2    ,		**UpperCamelCase: str    ,		)						->						Any:
											snake_case__													=       langs
											snake_case__													=       src_vocab_size
											snake_case__													=       tgt_vocab_size
											snake_case__													=       d_model  # encoder_embed_dim and decoder_embed_dim

											snake_case__													=       encoder_ffn_dim
											snake_case__													=       snake_case__													=       encoder_layers
											snake_case__													=       encoder_attention_heads
											snake_case__													=       encoder_layerdrop
											snake_case__													=       decoder_layerdrop
											snake_case__													=       decoder_ffn_dim
											snake_case__													=       decoder_layers
											snake_case__													=       decoder_attention_heads
											snake_case__													=       max_position_embeddings
											snake_case__													=       init_std  # Normal(0, this parameter)
											snake_case__													=       activation_function

											snake_case__													=       DecoderConfig(vocab_size=UpperCamelCase    ,		bos_token_id=UpperCamelCase  )
											if "decoder" in common_kwargs:
															del common_kwargs["decoder"]

											snake_case__													=       scale_embedding  # scale factor will be sqrt(d_model) if True

											# 3 Types of Dropout
											snake_case__													=       attention_dropout
											snake_case__													=       activation_dropout
											snake_case__													=       dropout

											snake_case__													=       use_cache
											super().__init__(
											    pad_token_id=UpperCamelCase    ,		bos_token_id=UpperCamelCase    ,		eos_token_id=UpperCamelCase    ,		decoder_start_token_id=UpperCamelCase    ,		is_encoder_decoder=UpperCamelCase    ,		tie_word_embeddings=UpperCamelCase    ,		forced_eos_token_id=UpperCamelCase    ,		max_length=UpperCamelCase    ,		num_beams=UpperCamelCase    ,		length_penalty=UpperCamelCase    ,		early_stopping=UpperCamelCase    ,		**UpperCamelCase    ,		)


							def lowerCAmelCase_	(       self: Any  )						->						Any:
											snake_case__													=       copy.deepcopy(self.__dict__  )
											snake_case__													=       self.decoder.to_dict()
											snake_case__													=       self.__class__.model_type
											return output












import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


__UpperCamelCase   :						int							=  [nn.LayerNorm]

__UpperCamelCase   :						List[str]							=  logging.get_logger(__name__)

__UpperCamelCase   :						str							=  version.parse(version.parse(torch.__version__).base_version)

__UpperCamelCase   :						List[str]							=  parsed_torch_version_base >= version.parse("""2.0""")
__UpperCamelCase   :						Dict							=  parsed_torch_version_base >= version.parse("""1.12""")
__UpperCamelCase   :						Tuple							=  parsed_torch_version_base >= version.parse("""1.11""")
__UpperCamelCase   :						Union[str, Any]							=  parsed_torch_version_base < version.parse("""1.11""")
def        a_	( _A ,				_A ,				_A ,				_A ,				self					)     ->     Any:







				"""simple docstring"""
				from torch import _softmax_backward_data

				if is_torch_less_than_1_11:
								return _softmax_backward_data(_A ,				_A ,				parent.dim ,				self					)
				else:
								return _softmax_backward_data(_A ,				_A ,				parent.dim ,				self.dtype					)
def        a_	( _A ,				_A ,				_A = 0					)     ->     nn.Linear:







				"""simple docstring"""
				snake_case__													=       index.to(layer.weight.device					)
				snake_case__													=       layer.weight.index_select(_A ,				_A					).clone().detach()
				if layer.bias is not None:
								if dim == 1:
												snake_case__													=       layer.bias.clone().detach()
								else:
												snake_case__													=       layer.bias[index].clone().detach()
				snake_case__													=       list(layer.weight.size()					)
				snake_case__													=       len(_A					)
				snake_case__													=       nn.Linear(new_size[1] ,				new_size[0] ,				bias=layer.bias is not None					).to(layer.weight.device					)
				snake_case__													=       False
				new_layer.weight.copy_(W.contiguous()					)
				snake_case__													=       True
				if layer.bias is not None:
								snake_case__													=       False
								new_layer.bias.copy_(b.contiguous()					)
								snake_case__													=       True
				return new_layer


class 					__SCREAMING_SNAKE_CASE(		nn.Module		):
							def __init__(       self: Union[str, Any]    ,		UpperCamelCase: Dict    ,		UpperCamelCase: List[Any]  )						->						Any:
											super().__init__()
											snake_case__													=       nf
											snake_case__													=       nn.Parameter(torch.empty(UpperCamelCase    ,		UpperCamelCase  )  )
											snake_case__													=       nn.Parameter(torch.zeros(UpperCamelCase  )  )
											nn.init.normal_(self.weight    ,		std=0.02  )


							def lowerCAmelCase_	(       self: Optional[Any]    ,		UpperCamelCase: int  )						->						List[Any]:
											snake_case__													=       x.size()[:-1] + (self.nf,)
											snake_case__													=       torch.addmm(self.bias    ,		x.view(-1    ,		x.size(-1  )  )    ,		self.weight  )
											snake_case__													=       x.view(UpperCamelCase  )
											return x


def        a_	( _A ,				_A ,				_A = 1					)     ->     ConvaD:







				"""simple docstring"""
				snake_case__													=       index.to(layer.weight.device					)
				snake_case__													=       layer.weight.index_select(_A ,				_A					).clone().detach()
				if dim == 0:
								snake_case__													=       layer.bias.clone().detach()
				else:
								snake_case__													=       layer.bias[index].clone().detach()
				snake_case__													=       list(layer.weight.size()					)
				snake_case__													=       len(_A					)
				snake_case__													=       ConvaD(new_size[1] ,				new_size[0]					).to(layer.weight.device					)
				snake_case__													=       False
				new_layer.weight.copy_(W.contiguous()					)
				snake_case__													=       True
				snake_case__													=       False
				new_layer.bias.copy_(b.contiguous()					)
				snake_case__													=       True
				return new_layer
def        a_	( _A ,				_A ,				_A = None					)     ->     Union[nn.Linear, ConvaD]:







				"""simple docstring"""
				if isinstance(_A ,				nn.Linear					):
								return prune_linear_layer(_A ,				_A ,				dim=0 if dim is None else dim					)
				elif isinstance(_A ,				_A					):
								return prune_convad_layer(_A ,				_A ,				dim=1 if dim is None else dim					)
				else:
								raise ValueError(f'''Can\'t prune layer of class {layer.__class__}'''					)
def        a_	( _A ,				_A ,				_A ,				*_A					)     ->     torch.Tensor:







				"""simple docstring"""
				assert len(_A					) > 0, f'''{input_tensors} has to be a tuple/list of tensors'''

				# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
				snake_case__													=       len(inspect.signature(_A					).parameters					)
				if num_args_in_forward_chunk_fn != len(_A					):
								raise ValueError(
								    f'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(_A					)} input '''
								    'tensors are given'					)

				if chunk_size > 0:
								snake_case__													=       input_tensors[0].shape[chunk_dim]
								for input_tensor in input_tensors:
												if input_tensor.shape[chunk_dim] != tensor_shape:
																raise ValueError(
																    f'''All input tenors have to be of the same shape: {tensor_shape}, '''
																    f'''found shape {input_tensor.shape[chunk_dim]}'''					)

								if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
												raise ValueError(
												    f'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
												    f'''size {chunk_size}'''					)

								snake_case__													=       input_tensors[0].shape[chunk_dim] // chunk_size

								# chunk input tensor into tuples
								snake_case__													=       tuple(input_tensor.chunk(_A ,				dim=_A					) for input_tensor in input_tensors					)
								# apply forward fn to every tuple
								snake_case__													=       tuple(forward_fn(*_A					) for input_tensors_chunk in zip(*_A					)					)
								# concatenate output at same dimension
								return torch.cat(_A ,				dim=_A					)

				return forward_fn(*_A					)
def        a_	( _A ,				_A ,				_A ,				_A					)     ->     Tuple[Set[int], torch.LongTensor]:







				"""simple docstring"""
				snake_case__													=       torch.ones(_A ,				_A					)
				snake_case__													=       set(_A					) - already_pruned_heads  # Convert to set and remove already pruned heads
				for head in heads:
								# Compute how many pruned heads are before the head and move the index accordingly
								snake_case__													=       head - sum(1 if h < head else 0 for h in already_pruned_heads					)
								snake_case__													=       0
				snake_case__													=       mask.view(-1					).contiguous().eq(1					)
				snake_case__													=       torch.arange(len(_A					)					)[mask].long()
				return heads, index
def        a_	( *_A ,				_A = None					)     ->     Tuple[torch.Tensor, ...]:







				"""simple docstring"""
				return torch.meshgrid(*_A ,				indexing=_A					)


def        a_	( _A					)     ->     Tuple[torch.device, int, int]:







				"""simple docstring"""
				return tensor.device, storage_ptr(_A					), storage_size(_A					)




