"""simple docstring"""






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
						import torch

lowerCAmelCase__  :  str											=     logging.get_logger(__name__)



def   a_						( ):
				return importlib.util.find_spec('deepspeed'	) is not None


if is_accelerate_available() and is_deepspeed_available():
						from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
						# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
						# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
						from builtins import object as DeepSpeedConfig
class     snake_case	(			__UpperCAmelCase    ):




					"""simple docstring"""

					def __init__(    self       :     Optional[Any]   ,lowerCamelCase__       :     int      ):
									# set global weakref object
									set_hf_deepspeed_config(self      )
									dep_version_check('accelerate'      )
									dep_version_check('deepspeed'      )
									super().__init__(lowerCamelCase__      )






class     snake_case	(			__UpperCAmelCase    ):




					"""simple docstring"""

					def __init__(    self       :     Dict   ,lowerCamelCase__       :     List[str]      ):
									super().__init__(lowerCamelCase__      )
									UpperCAmelCase__					=							None
									UpperCAmelCase__					=							[]







					def 		__lowerCAmelCase						(    self       :     List[str]      ):
									if self._dtype is None:
													raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype'      )
									return self._dtype







					def 		__lowerCAmelCase						(    self       :     Union[str, Any]   ,lowerCamelCase__       :     Dict      ):
									UpperCAmelCase__					=							self.get_value(lowerCamelCase__      )
									if val is None:
													return False
									else:
													return val == "auto"







					def 		__lowerCAmelCase						(    self       :     Optional[Any]   ,lowerCamelCase__       :     Optional[Any]   ,lowerCamelCase__       :     List[Any]   ,lowerCamelCase__       :     Tuple=None   ,lowerCamelCase__       :     Any=True      ):
									UpperCAmelCase__				,     UpperCAmelCase__					=							self.find_config_node(lowerCamelCase__      )
									if config is None:
													return

									if config.get(lowerCamelCase__      ) == "auto":
													UpperCAmelCase__					=							hf_val
													return

									if not must_match:
													return

									UpperCAmelCase__					=							config.get(lowerCamelCase__      )
									if ds_val is not None and ds_val != hf_val:
													self.mismatches.append(f'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''      )

					snake_case__       =			partialmethod(__UpperCAmelCase				,   must_match=__UpperCAmelCase    )







					def 		__lowerCAmelCase						(    self       :     Any   ,lowerCamelCase__       :     Union[str, Any]      ):
									UpperCAmelCase__					=							args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
									self.fill_match(
									    'train_micro_batch_size_per_gpu'   ,args.per_device_train_batch_size   ,'per_device_train_batch_size'      )
									self.fill_match('gradient_accumulation_steps'   ,args.gradient_accumulation_steps   ,'gradient_accumulation_steps'      )
									self.fill_match('train_batch_size'   ,lowerCamelCase__   ,'train_batch_size (calculated)'      )
									self.fill_match('gradient_clipping'   ,args.max_grad_norm   ,'max_grad_norm'      )

									self.fill_match('optimizer.params.lr'   ,args.learning_rate   ,'learning_rate'      )
									self.fill_match('optimizer.params.betas'   ,[args.adam_betaa, args.adam_betaa]   ,'adam_beta1+adam_beta2'      )
									self.fill_match('optimizer.params.eps'   ,args.adam_epsilon   ,'adam_epsilon'      )
									self.fill_match('optimizer.params.weight_decay'   ,args.weight_decay   ,'weight_decay'      )

									self.fill_only('scheduler.params.warmup_min_lr'   ,0      )  # not a trainer arg
									self.fill_match('scheduler.params.warmup_max_lr'   ,args.learning_rate   ,'learning_rate'      )
									# total_num_steps - will get set in trainer_config_finalize

									# fp16
									if args.fpaa or args.fpaa_full_eval:
													UpperCAmelCase__					=							'apex' if args.fpaa_backend == 'apex' else 'amp'
									else:
													UpperCAmelCase__					=							None

									if args.save_on_each_node:
													# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
													UpperCAmelCase__					=							self.config.get('checkpoint'   ,{}      )
													UpperCAmelCase__					=							args.save_on_each_node

									# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
									# any here unless the user did the work
									self.fill_match(
									    'fp16.enabled'   ,((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')   ,'fp16|fp16_full_eval+fp16_backend(amp)'   ,)

									# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
									# ZeRO features
									self.fill_match('amp.enabled'   ,fpaa_backend == 'apex'   ,'fp16+fp16_backend(apex)'      )
									self.fill_match('amp.opt_level'   ,args.fpaa_opt_level   ,'fp16_opt_level'      )

									self.fill_match('bf16.enabled'   ,(args.bfaa or args.bfaa_full_eval)   ,'bf16|bf16_full_eval'      )

									# deepspeed's default mode is fp16 unless there is a config that says differently
									if self.is_true('bf16.enabled'      ):
													UpperCAmelCase__					=							torch.bfloataa
									elif self.is_false('fp16.enabled'      ):
													UpperCAmelCase__					=							torch.floataa
									else:
													UpperCAmelCase__					=							torch.floataa







					def 		__lowerCAmelCase						(    self       :     Any   ,lowerCamelCase__       :     Optional[Any]   ,lowerCamelCase__       :     Optional[int]   ,lowerCamelCase__       :     List[Any]      ):
									UpperCAmelCase__					=							[
									    'zero_optimization.reduce_bucket_size',
									    'zero_optimization.stage3_prefetch_bucket_size',
									    'zero_optimization.stage3_param_persistence_threshold',
									]
									UpperCAmelCase__					=							[x for x in hidden_size_based_keys if self.is_auto(lowerCamelCase__      )]

									if len(lowerCamelCase__      ) > 0:
													if hasattr(model.config   ,'hidden_size'      ):
																	UpperCAmelCase__					=							model.config.hidden_size
													elif hasattr(model.config   ,'hidden_sizes'      ):
																	# if there are many hidden sizes pick the largest one
																	UpperCAmelCase__					=							max(model.config.hidden_sizes      )
													else:
																	raise ValueError(
																	    'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
																	    'therefore it\'s not possible to automatically fill out the following `auto` entries '
																	    f'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
																	    '`auto` values for these keys with an integer value of your choice.'      )

													self.fill_only('zero_optimization.reduce_bucket_size'   ,hidden_size * hidden_size      )
													if self.is_zeroa():
																	# automatically assign the optimal config values based on model config
																	self.fill_only('zero_optimization.stage3_prefetch_bucket_size'   ,0.9 * hidden_size * hidden_size      )
																	self.fill_only('zero_optimization.stage3_param_persistence_threshold'   ,10 * hidden_size      )

        # scheduler
									self.fill_match('scheduler.params.total_num_steps'   ,lowerCamelCase__   ,'num_training_steps (calculated)'      )
									self.fill_match('scheduler.params.warmup_num_steps'   ,args.get_warmup_steps(lowerCamelCase__      )   ,'warmup_steps'      )

									if len(self.mismatches      ) > 0:
													UpperCAmelCase__					=							'\n'.join(self.mismatches      )
													raise ValueError(
													    'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
													    f''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''      )






lowerCAmelCase__  :  List[str]											=     None



def   a_						( lowerCamelCase	):
				# this is a special weakref global object to allow us to get to Deepspeed config from APIs
				# that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
				global _hf_deepspeed_config_weak_ref
				# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
				UpperCAmelCase__					=							weakref.ref(lowerCamelCase	)



def   a_						( ):
				# useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
				global _hf_deepspeed_config_weak_ref
				UpperCAmelCase__					=							None



def   a_						( ):
				if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
								return _hf_deepspeed_config_weak_ref().is_zeroa()
				else:
								return False



def   a_						( ):
				if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
								return _hf_deepspeed_config_weak_ref().config
				else:
								return None



def   a_						( lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase	):
				from accelerate.utils import DummyOptim, DummyScheduler

				UpperCAmelCase__					=							hf_deepspeed_config.config

				# Optimizer + Scheduler
				# Currently supported combos:
				# 1. DS scheduler + DS optimizer: Yes
				# 2. HF scheduler + HF optimizer: Yes
				# 3. DS scheduler + HF optimizer: Yes
				# 4. HF scheduler + DS optimizer: No
				#
				# Unless Offload is enabled in which case it's:
				# 1. DS scheduler + DS optimizer: Yes
				# 2. HF scheduler + HF optimizer: Mostly*
				# 3. DS scheduler + HF optimizer: Mostly*
				# 4. HF scheduler + DS optimizer: No
				#
				# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

				UpperCAmelCase__					=							None
				if "optimizer" in config:
								if args.adafactor:
												raise ValueError(
												    '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
												    'Only one optimizer can be configured.'	)
								UpperCAmelCase__					=							DummyOptim(params=lowerCamelCase	)
				else:
								if hf_deepspeed_config.is_offload():
												logger.info(
												    'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
												    ' custom optimizer has both CPU and GPU implementation (except LAMB)'	)

								# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
								# But trainer uses AdamW by default.
								UpperCAmelCase__					=							trainer.create_optimizer()
								# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
								UpperCAmelCase__					=							True

				UpperCAmelCase__					=							None
				if "scheduler" in config:
								UpperCAmelCase__					=							DummyScheduler(lowerCamelCase	)
				else:
								if isinstance(lowerCamelCase ,      lowerCamelCase	):
												raise ValueError(
												    'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
												    'Please configure a scheduler in the DeepSpeed config.'	)
								UpperCAmelCase__					=							trainer.create_scheduler(num_training_steps=lowerCamelCase ,      optimizer=lowerCamelCase	)

				return optimizer, lr_scheduler



def   a_						( lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase=False	):
				from deepspeed.utils import logger as ds_logger

				UpperCAmelCase__					=							trainer.model
				UpperCAmelCase__					=							trainer.args

				UpperCAmelCase__					=							trainer.accelerator.state.deepspeed_plugin.hf_ds_config

				# resume config update - some bits like `model` and `num_training_steps` only become available during train
				hf_deepspeed_config.trainer_config_finalize(lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase	)

				# set the Deepspeed log level consistent with the Trainer
				ds_logger.setLevel(args.get_process_log_level()	)

				if inference:
								# only Z3 makes sense for the inference
								if not hf_deepspeed_config.is_zeroa():
												raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'	)

								# in case the training config is re-used for inference
								hf_deepspeed_config.del_config_sub_tree('optimizer'	)
								hf_deepspeed_config.del_config_sub_tree('lr_scheduler'	)
								UpperCAmelCase__				,     UpperCAmelCase__					=							None, None
								UpperCAmelCase__					=							None
				else:
								UpperCAmelCase__					=							None  # important for when deepspeed_init is used as re-init
								UpperCAmelCase__					=							list(filter(lambda lowerCamelCase	: p.requires_grad ,      model.parameters()	)	)
								UpperCAmelCase__				,     UpperCAmelCase__					=							deepspeed_optim_sched(
								    lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase	)

				# keep for quick debug:
				# from pprint import pprint; pprint(config)

				return optimizer, lr_scheduler







def   a_						( lowerCamelCase ,      lowerCamelCase	):
				# it's possible that the user is trying to resume from model_path, which doesn't necessarily
				# contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
				# a resume from a checkpoint and not just a local pretrained weight. So we check here if the
				# path contains what looks like a deepspeed checkpoint
				import glob

				UpperCAmelCase__					=							sorted(glob.glob(f'''{checkpoint_path}/global_step*'''	)	)

				if len(lowerCamelCase	) > 0:
								logger.info(f'''Attempting to resume from {checkpoint_path}'''	)
								# this magically updates self.optimizer and self.lr_scheduler
								UpperCAmelCase__				,     UpperCAmelCase__					=							deepspeed_engine.load_checkpoint(
								    lowerCamelCase ,      load_optimizer_states=lowerCamelCase ,      load_lr_scheduler_states=lowerCamelCase	)
								if load_path is None:
												raise ValueError(f'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''	)
				else:
								raise ValueError(f'''Can\'t find a valid checkpoint at {checkpoint_path}'''	)



"""simple docstring"""






import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
						import torch


lowerCAmelCase__  :  Optional[Any]											=     logging.get_logger(__name__)
class     snake_case	:




					"""simple docstring"""

					def __init__(    self       :     Optional[Any]   ,lowerCamelCase__       :     Dict   ,lowerCamelCase__       :     Optional[Any]=21   ,lowerCamelCase__       :     str=[]   ,lowerCamelCase__       :     Any=None      ):
									UpperCAmelCase__					=							model
									UpperCAmelCase__					=							trace_batch_nums
									UpperCAmelCase__					=							abort_after_batch_num

									# keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
									UpperCAmelCase__					=							collections.deque([]   ,lowerCamelCase__      )
									UpperCAmelCase__					=							[]
									UpperCAmelCase__					=							0
									UpperCAmelCase__					=							0
									UpperCAmelCase__					=							False
									UpperCAmelCase__					=							'                 '

									self.analyse_model()

									self.register_forward_hook()







					def 		__lowerCAmelCase						(    self       :     Union[str, Any]   ,lowerCamelCase__       :     Tuple=None      ):
									if frame is not None:
													self.expand_frame(lowerCamelCase__      )
									self.frames.append('\n'.join(self.frame      )      )
									UpperCAmelCase__					=							[]  # start a new frame







					def 		__lowerCAmelCase						(    self       :     int   ,lowerCamelCase__       :     int      ):
									self.frame.append(lowerCamelCase__      )







					def 		__lowerCAmelCase						(    self       :     List[str]      ):
									print('\n'.join(self.frames      )      )
									UpperCAmelCase__					=							[]







					def 		__lowerCAmelCase						(    self       :     Tuple      ):
									UpperCAmelCase__					=							[]







					def 		__lowerCAmelCase						(    self       :     Dict      ):
									print(f'''\nDetected inf/nan during batch_number={self.batch_number}'''      )
									print(f'''Last {len(self.frames      )} forward frames:'''      )
									print(f'''{"abs min":8} {"abs max":8} metadata'''      )
									print('\n'.join(self.frames      )      )
									print('\n\n'      )
									UpperCAmelCase__					=							[]







					def 		__lowerCAmelCase						(    self       :     Tuple      ):
									# extract the fully qualified module names, to be able to report at run time. e.g.:
									# encoder.block.2.layer.0.SelfAttention.o
									#
									# for shared weights only the first shared module name will be registered
									UpperCAmelCase__					=							{m: name for name, m in self.model.named_modules()}
									# self.longest_module_name = max(len(v) for v in self.module_names.values())







					def 		__lowerCAmelCase						(    self       :     Optional[Any]   ,lowerCamelCase__       :     int   ,lowerCamelCase__       :     List[Any]      ):
									if torch.is_tensor(lowerCamelCase__      ):
													self.expand_frame(get_abs_min_max(lowerCamelCase__   ,lowerCamelCase__      )      )
													if detect_overflow(lowerCamelCase__   ,lowerCamelCase__      ):
																	UpperCAmelCase__					=							True
									elif var is None:
													self.expand_frame(f'''{"None":>17} {ctx}'''      )
									else:
													self.expand_frame(f'''{"not a tensor":>17} {ctx}'''      )







					def 		__lowerCAmelCase						(    self       :     Optional[int]      ):
									self.expand_frame(f'''\n\n{self.prefix} *** Starting batch number={self.batch_number} ***'''      )
									self.expand_frame(f'''{"abs min":8} {"abs max":8} metadata'''      )







					def 		__lowerCAmelCase						(    self       :     Optional[int]      ):
									self.expand_frame(f'''{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n'''      )







					def 		__lowerCAmelCase						(    self       :     List[str]   ,lowerCamelCase__       :     List[str]   ,lowerCamelCase__       :     List[Any]   ,lowerCamelCase__       :     Optional[Any]      ):
									self.expand_frame(f'''{self.prefix} {self.module_names[module]} {module.__class__.__name__}'''      )

									# params
									for name, p in module.named_parameters(recurse=lowerCamelCase__      ):
													self.analyse_variable(lowerCamelCase__   ,lowerCamelCase__      )

									# inputs
									if isinstance(lowerCamelCase__   ,lowerCamelCase__      ):
													for i, x in enumerate(lowerCamelCase__      ):
																	self.analyse_variable(lowerCamelCase__   ,f'''input[{i}]'''      )
									else:
													self.analyse_variable(lowerCamelCase__   ,'input'      )

									# outputs
									if isinstance(lowerCamelCase__   ,lowerCamelCase__      ):
													for i, x in enumerate(lowerCamelCase__      ):
																	# possibly a tuple of tuples
																	if isinstance(lowerCamelCase__   ,lowerCamelCase__      ):
																					for j, y in enumerate(lowerCamelCase__      ):
																									self.analyse_variable(lowerCamelCase__   ,f'''output[{i}][{j}]'''      )
																	else:
																					self.analyse_variable(lowerCamelCase__   ,f'''output[{i}]'''      )
									else:
													self.analyse_variable(lowerCamelCase__   ,'output'      )

									self.save_frame()







					def 		__lowerCAmelCase						(    self       :     Union[str, Any]      ):
									self.model.apply(self._register_forward_hook      )







					def 		__lowerCAmelCase						(    self       :     List[Any]   ,lowerCamelCase__       :     str      ):
									module.register_forward_hook(self.forward_hook      )







					def 		__lowerCAmelCase						(    self       :     Optional[Any]   ,lowerCamelCase__       :     Any   ,lowerCamelCase__       :     str   ,lowerCamelCase__       :     Union[str, Any]      ):
									# - input is a tuple of packed inputs (could be non-Tensors)
									# - output could be a Tensor or a tuple of Tensors and non-Tensors

									UpperCAmelCase__					=							False

									UpperCAmelCase__					=							True if self.batch_number in self.trace_batch_nums else False
									if trace_mode:
													self.reset_saved_frames()

									if self.total_calls == 0:
													self.batch_start_frame()
									self.total_calls += 1

									# count batch numbers - the very first forward hook of the batch will be called when the
									# batch completes - i.e. it gets called very last - we know this batch has finished
									if module == self.model:
													self.batch_number += 1
													UpperCAmelCase__					=							True

									self.create_frame(lowerCamelCase__   ,lowerCamelCase__   ,lowerCamelCase__      )

									# if last_frame_of_batch:
									#     self.batch_end_frame()

									if trace_mode:
													self.trace_frames()

									if last_frame_of_batch:
													self.batch_start_frame()

									if self.detected_overflow and not trace_mode:
													self.dump_saved_frames()

													# now we can abort, as it's pointless to continue running
													raise ValueError(
													    'DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. '
													    'Please scroll up above this traceback to see the activation values prior to this event.'      )

									# abort after certain batch if requested to do so
									if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
													raise ValueError(
													    f'''DebugUnderflowOverflow: aborting after {self.batch_number} batches due to'''
													    f''' `abort_after_batch_num={self.abort_after_batch_num}` arg'''      )






def   a_						( lowerCamelCase ,      lowerCamelCase	):
				UpperCAmelCase__					=							var.abs()
				return f'''{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}'''



def   a_						( lowerCamelCase ,      lowerCamelCase	):
				UpperCAmelCase__					=							False
				if torch.isnan(lowerCamelCase	).any().item():
								UpperCAmelCase__					=							True
								print(f'''{ctx} has nans'''	)
				if torch.isinf(lowerCamelCase	).any().item():
								UpperCAmelCase__					=							True
								print(f'''{ctx} has infs'''	)

				# if needed to monitor large elements can enable the following
				if 0:  # and detected:
								UpperCAmelCase__					=							var[torch.ge(var.abs() ,      1_0_0	)]
								if naaa.numel() > 0:
												print(f'''{ctx}:  n100={naaa.numel()}'''	)
								UpperCAmelCase__					=							var[torch.ge(var.abs() ,      1_0_0_0	)]
								if naaaa.numel() > 0:
												print(f'''{ctx}: n1000={naaaa.numel()}'''	)
								UpperCAmelCase__					=							var[torch.ge(var.abs() ,      1_0_0_0_0	)]
								if naaaaa.numel() > 0:
												print(f'''{ctx}: n10000={naaaaa.numel()}'''	)

				if 0:
								print(f'''min={var.min():9.2e} max={var.max():9.2e}'''	)

				if 0:
								print(f'''min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})'''	)

				return detected







class     snake_case	(			__UpperCAmelCase    ):




					"""simple docstring"""

					snake_case__       =			"underflow_overflow"
					snake_case__       =			"tpu_metrics_debug"



