
"""simple docstring"""




import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


snake_case			       =    [nn.LayerNorm]

snake_case			       =    logging.get_logger(__name__)

snake_case			       =    version.parse(version.parse(torch.__version__).base_version)

snake_case			       =    parsed_torch_version_base >= version.parse('2.0')
snake_case			       =    parsed_torch_version_base >= version.parse('1.12')
snake_case			       =    parsed_torch_version_base >= version.parse('1.11')
snake_case			       =    parsed_torch_version_base < version.parse('1.11')

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					self      ):
					from torch import _softmax_backward_data

					if is_torch_less_than_1_11:
										return _softmax_backward_data(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					parent.dim,					self      )
					else:
										return _softmax_backward_data(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					parent.dim,					self.dtype      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_ = 0      ):
					SCREAMING_SNAKE_CASE           =     index.to(layer.weight.device      )
					SCREAMING_SNAKE_CASE           =     layer.weight.index_select(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ).clone().detach()
					if layer.bias is not None:
										if dim == 1:
															SCREAMING_SNAKE_CASE           =     layer.bias.clone().detach()
										else:
															SCREAMING_SNAKE_CASE           =     layer.bias[index].clone().detach()
					SCREAMING_SNAKE_CASE           =     list(layer.weight.size()      )
					SCREAMING_SNAKE_CASE           =     len(SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     nn.Linear(new_size[1],					new_size[0],					bias=layer.bias is not None      ).to(layer.weight.device      )
					SCREAMING_SNAKE_CASE           =     False
					new_layer.weight.copy_(W.contiguous()      )
					SCREAMING_SNAKE_CASE           =     True
					if layer.bias is not None:
										SCREAMING_SNAKE_CASE           =     False
										new_layer.bias.copy_(b.contiguous()      )
										SCREAMING_SNAKE_CASE           =     True
					return new_layer




class 					UpperCamelCase     (	nn.Module				):

				"""simple docstring"""



				def __init__(    self		,     lowercase__		,     lowercase__      )   ->   List[Any]:
									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE           =     nf
									SCREAMING_SNAKE_CASE           =     nn.Parameter(torch.empty(lowercase__		,     lowercase__      )      )
									SCREAMING_SNAKE_CASE           =     nn.Parameter(torch.zeros(lowercase__      )      )
									nn.init.normal_(self.weight		,     std=0.02      )




				def 							A (    self		,     lowercase__      )   ->   Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     x.size()[:-1] + (self.nf,)
									SCREAMING_SNAKE_CASE           =     torch.addmm(self.bias		,     x.view(-1		,     x.size(-1      )      )		,     self.weight      )
									SCREAMING_SNAKE_CASE           =     x.view(lowercase__      )
									return x






def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_ = 1      ):
					SCREAMING_SNAKE_CASE           =     index.to(layer.weight.device      )
					SCREAMING_SNAKE_CASE           =     layer.weight.index_select(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ).clone().detach()
					if dim == 0:
										SCREAMING_SNAKE_CASE           =     layer.bias.clone().detach()
					else:
										SCREAMING_SNAKE_CASE           =     layer.bias[index].clone().detach()
					SCREAMING_SNAKE_CASE           =     list(layer.weight.size()      )
					SCREAMING_SNAKE_CASE           =     len(SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     ConvaD(new_size[1],					new_size[0]      ).to(layer.weight.device      )
					SCREAMING_SNAKE_CASE           =     False
					new_layer.weight.copy_(W.contiguous()      )
					SCREAMING_SNAKE_CASE           =     True
					SCREAMING_SNAKE_CASE           =     False
					new_layer.bias.copy_(b.contiguous()      )
					SCREAMING_SNAKE_CASE           =     True
					return new_layer

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_ = None      ):
					if isinstance(SCREAMING_SNAKE_CASE_,					nn.Linear      ):
										return prune_linear_layer(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					dim=0 if dim is None else dim      )
					elif isinstance(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
										return prune_convad_layer(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					dim=1 if dim is None else dim      )
					else:
										raise ValueError(f'''Can\'t prune layer of class {layer.__class__}'''      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					*SCREAMING_SNAKE_CASE_      ):
					assert len(SCREAMING_SNAKE_CASE_      ) > 0, f'''{input_tensors} has to be a tuple/list of tensors'''

					# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
					SCREAMING_SNAKE_CASE           =     len(inspect.signature(SCREAMING_SNAKE_CASE_      ).parameters      )
					if num_args_in_forward_chunk_fn != len(SCREAMING_SNAKE_CASE_      ):
										raise ValueError(
										    f'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(SCREAMING_SNAKE_CASE_      )} input '''
										    'tensors are given'      )

					if chunk_size > 0:
										SCREAMING_SNAKE_CASE           =     input_tensors[0].shape[chunk_dim]
										for input_tensor in input_tensors:
															if input_tensor.shape[chunk_dim] != tensor_shape:
																				raise ValueError(
																				    f'''All input tenors have to be of the same shape: {tensor_shape}, '''
																				    f'''found shape {input_tensor.shape[chunk_dim]}'''      )

										if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
															raise ValueError(
															    f'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
															    f'''size {chunk_size}'''      )

										SCREAMING_SNAKE_CASE           =     input_tensors[0].shape[chunk_dim] // chunk_size

										# chunk input tensor into tuples
										SCREAMING_SNAKE_CASE           =     tuple(input_tensor.chunk(SCREAMING_SNAKE_CASE_,					dim=SCREAMING_SNAKE_CASE_      ) for input_tensor in input_tensors      )
										# apply forward fn to every tuple
										SCREAMING_SNAKE_CASE           =     tuple(forward_fn(*SCREAMING_SNAKE_CASE_      ) for input_tensors_chunk in zip(*SCREAMING_SNAKE_CASE_      )      )
										# concatenate output at same dimension
										return torch.cat(SCREAMING_SNAKE_CASE_,					dim=SCREAMING_SNAKE_CASE_      )

					return forward_fn(*SCREAMING_SNAKE_CASE_      )

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      ):
					SCREAMING_SNAKE_CASE           =     torch.ones(SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_      )
					SCREAMING_SNAKE_CASE           =     set(SCREAMING_SNAKE_CASE_      ) - already_pruned_heads  # Convert to set and remove already pruned heads
					for head in heads:
										# Compute how many pruned heads are before the head and move the index accordingly
										SCREAMING_SNAKE_CASE           =     head - sum(1 if h < head else 0 for h in already_pruned_heads      )
										SCREAMING_SNAKE_CASE           =     0
					SCREAMING_SNAKE_CASE           =     mask.view(-1      ).contiguous().eq(1      )
					SCREAMING_SNAKE_CASE           =     torch.arange(len(SCREAMING_SNAKE_CASE_      )      )[mask].long()
					return heads, index

def   UpperCamelCase_  (		*SCREAMING_SNAKE_CASE_,					SCREAMING_SNAKE_CASE_ = None      ):
					return torch.meshgrid(*SCREAMING_SNAKE_CASE_,					indexing=SCREAMING_SNAKE_CASE_      )



def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_      ):
					return tensor.device, storage_ptr(SCREAMING_SNAKE_CASE_      ), storage_size(SCREAMING_SNAKE_CASE_      )


"""simple docstring"""




import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
	import torch

	from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
	import pyanvml.pyanvml as nvml


snake_case			       =		logging.get_logger(__name__)




class 					UpperCamelCase     (	__magic_name__				):

				"""simple docstring"""



				UpperCAmelCase_    :				PyTorchBenchmarkArguments
				UpperCAmelCase_    :				PretrainedConfig
				UpperCAmelCase_    :				str						  =    "PyTorch"


				@property
				def 							A (    self      )   ->   List[str]:
									"""simple docstring"""







									return torch.__version__


				def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__      )   ->   float:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self._prepare_inference_func(lowercase__		,     lowercase__		,     lowercase__      )
									return self._measure_speed(_inference      )


				def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__      )   ->   [Memory, Optional[MemorySummary]]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self._prepare_inference_func(lowercase__		,     lowercase__		,     lowercase__      )
									return self._measure_memory(_inference      )


				def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__      )   ->   float:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self._prepare_train_func(lowercase__		,     lowercase__		,     lowercase__      )
									return self._measure_speed(_train      )


				def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__      )   ->   [Memory, Optional[MemorySummary]]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self._prepare_train_func(lowercase__		,     lowercase__		,     lowercase__      )
									return self._measure_memory(_train      )


				def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__      )   ->   Callable[[], None]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.config_dict[model_name]

									if self.args.torchscript:
														SCREAMING_SNAKE_CASE           =     True

									SCREAMING_SNAKE_CASE           =     (
									    hasattr(lowercase__		,     'architectures'      )
									    and isinstance(config.architectures		,     lowercase__      )
									    and len(config.architectures      ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																			SCREAMING_SNAKE_CASE           =     config.architectures[0]
																			SCREAMING_SNAKE_CASE           =     __import__('transformers'		,     fromlist=[model_class]      )
																			SCREAMING_SNAKE_CASE           =     getattr(lowercase__		,     lowercase__      )
																			SCREAMING_SNAKE_CASE           =     model_cls(lowercase__      )
														except ImportError:
																			raise ImportError(
																			    f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
																			    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'      )
									else:
														SCREAMING_SNAKE_CASE           =     MODEL_MAPPING[config.__class__](lowercase__      )

									model.eval()
									model.to(self.args.device      )

									# encoder-decoder has vocab size saved differently
									SCREAMING_SNAKE_CASE           =     config.vocab_size if hasattr(lowercase__		,     'vocab_size'      ) else config.encoder.vocab_size
									SCREAMING_SNAKE_CASE           =     torch.randint(lowercase__		,     (batch_size, sequence_length)		,     dtype=torch.long		,     device=self.args.device      )

									if self.args.fpaa:
														logger.info('Running training in Mixed Precision...'      )
														if not self.args.is_gpu:
																			raise ValueError('Mixed precision is possible only for GPU.'      )
														# amp seems to have memory leaks so that memory usage
														# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
														model.half()

									if self.args.torchscript:
														with torch.no_grad():
																			SCREAMING_SNAKE_CASE           =     torch.jit.trace(lowercase__		,     lowercase__      )
									else:
														SCREAMING_SNAKE_CASE           =     model

									def encoder_decoder_forward():
														with torch.no_grad():
																			SCREAMING_SNAKE_CASE           =     inference_model(lowercase__		,     decoder_input_ids=lowercase__      )
														return outputs

									def encoder_forward():
														with torch.no_grad():
																			SCREAMING_SNAKE_CASE           =     inference_model(lowercase__      )
														return outputs

									SCREAMING_SNAKE_CASE           =     encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
									return _forward


				def 							A (    self		,     lowercase__		,     lowercase__		,     lowercase__      )   ->   Callable[[], None]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE           =     self.config_dict[model_name]

									SCREAMING_SNAKE_CASE           =     (
									    hasattr(lowercase__		,     'architectures'      )
									    and isinstance(config.architectures		,     lowercase__      )
									    and len(config.architectures      ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																			SCREAMING_SNAKE_CASE           =     config.architectures[0]
																			SCREAMING_SNAKE_CASE           =     __import__('transformers'		,     fromlist=[model_class]      )
																			SCREAMING_SNAKE_CASE           =     getattr(lowercase__		,     lowercase__      )
																			SCREAMING_SNAKE_CASE           =     model_cls(lowercase__      )
														except ImportError:
																			raise ImportError(
																			    f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
																			    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'      )
									else:
														SCREAMING_SNAKE_CASE           =     MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowercase__      )

									if self.args.torchscript:
														raise NotImplementedError('Training for torchscript is currently not implemented'      )
									else:
														SCREAMING_SNAKE_CASE           =     model

									model.train()
									model.to(self.args.device      )

									# encoder-decoder has vocab size saved differently
									SCREAMING_SNAKE_CASE           =     config.vocab_size if hasattr(lowercase__		,     'vocab_size'      ) else config.encoder.vocab_size
									SCREAMING_SNAKE_CASE           =     torch.randint(lowercase__		,     (batch_size, sequence_length)		,     dtype=torch.long		,     device=self.args.device      )

									if self.args.fpaa:
														logger.info('Running training in Mixed Precision...'      )
														if not self.args.is_gpu:
																			raise ValueError('Mixed precision is possible only for GPU.'      )

														# amp seems to have memory leaks so that memory usage
														# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
														model.half()

									def compute_loss_and_backprob_encoder():
														SCREAMING_SNAKE_CASE           =     train_model(lowercase__		,     labels=lowercase__      )[0]
														loss.backward()
														return loss

									def compute_loss_and_backprob_encoder_decoder():
														SCREAMING_SNAKE_CASE           =     train_model(lowercase__		,     decoder_input_ids=lowercase__		,     labels=lowercase__      )[0]
														loss.backward()
														return loss

									SCREAMING_SNAKE_CASE           =     (
									    compute_loss_and_backprob_encoder_decoder
									    if config.is_encoder_decoder
									    else compute_loss_and_backprob_encoder
									)
									return _train


				def 							A (    self		,     lowercase__      )   ->   float:
									"""simple docstring"""







									try:
														if self.args.is_tpu or self.args.torchscript:
																			# run additional 10 times to stabilize compilation for tpu and torchscript
																			logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'      )
																			timeit.repeat(
																			    lowercase__		,     repeat=1		,     number=5		,     )

														# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
														SCREAMING_SNAKE_CASE           =     timeit.repeat(
														    lowercase__		,     repeat=self.args.repeat		,     number=10		,     )

														if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
																			import torch_xla.debug.metrics as met

																			self.print_fn(met.metrics_report()      )

														return min(lowercase__      ) / 10.0
									except RuntimeError as e:
														self.print_fn(f'''Doesn\'t fit on GPU. {e}'''      )
														return "N/A"




				def 							A (    self		,     lowercase__      )   ->   [Memory, MemorySummary]:
									"""simple docstring"""







									try:
														if self.args.trace_memory_line_by_line:
																			SCREAMING_SNAKE_CASE           =     start_memory_tracing('transformers'      )

														if self.args.is_tpu:
																			# tpu
																			raise NotImplementedError(
																			    'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
																			    ' `--no-memory` or `args.memory=False`'      )
														elif self.args.is_gpu:
																			if not is_pyanvml_available():
																								logger.warning(
																								    'py3nvml not installed, we won\'t log GPU memory usage. '
																								    'Install py3nvml (pip install py3nvml) to log information about GPU.'      )
																								SCREAMING_SNAKE_CASE           =     'N/A'
																			else:
																								logger.info(
																								    'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
																								    ' on the same GPU.'      )
																								# init nvml
																								nvml.nvmlInit()
																								func()
																								SCREAMING_SNAKE_CASE           =     nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx      )
																								SCREAMING_SNAKE_CASE           =     nvml.nvmlDeviceGetMemoryInfo(lowercase__      )
																								SCREAMING_SNAKE_CASE           =     meminfo.used
																								SCREAMING_SNAKE_CASE           =     Memory(lowercase__      )
																								# shutdown nvml
																								nvml.nvmlShutdown()
														else:
																			# cpu
																			SCREAMING_SNAKE_CASE           =     measure_peak_memory_cpu(lowercase__      )
																			SCREAMING_SNAKE_CASE           =     Memory(lowercase__      ) if isinstance(lowercase__		,     lowercase__      ) else memory_bytes

														if self.args.trace_memory_line_by_line:
																			SCREAMING_SNAKE_CASE           =     stop_memory_tracing(lowercase__      )
														else:
																			SCREAMING_SNAKE_CASE           =     None

														return memory, summary
									except RuntimeError as e:
														self.print_fn(f'''Doesn\'t fit on GPU. {e}'''      )
														return "N/A", None
