

import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
	import torch

__snake_case							:     Dict 						=	logging.get_logger(__name__)


def 			_UpperCAmelCase							(		) ->    Optional[Any]:

			'''simple docstring'''



			return importlib.util.find_spec("""deepspeed""") is not None


if is_accelerate_available() and is_deepspeed_available():
	from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
	# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
	# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
	from builtins import object as DeepSpeedConfig



class    A__(a_    ):
						"""simple docstring"""






						def __init__(  self       ,     _lowercase  )      ->				Any:
									set_hf_deepspeed_config(self  )
									dep_version_check("""accelerate"""  )
									dep_version_check("""deepspeed"""  )
									super().__init__(snake_case__  )







class    A__(a_    ):
						"""simple docstring"""






						def __init__(  self       ,     _lowercase  )      ->				Union[str, Any]:
									super().__init__(snake_case__  )
									a_		:		Optional[int]				    =				None
									a_		:		int				    =				[]





						def 	UpperCamelCase__			(  self  )      ->				List[Any]:


									if self._dtype is None:
												raise ValueError("""trainer_config_process() wasn't called yet to tell dtype"""  )
									return self._dtype





						def 	UpperCamelCase__			(  self       ,     _lowercase  )      ->				Any:
									a_		:		Dict				    =				self.get_value(snake_case__  )
									if val is None:
												return False
									else:
												return val == "auto"





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase=None       ,     _lowercase=True  )      ->				Any:
									a_    ,			a_		:		Optional[Any]				    =				self.find_config_node(snake_case__  )
									if config is None:
												return

									if config.get(snake_case__  ) == "auto":
												a_		:		List[str]				    =				hf_val
												return

									if not must_match:
												return

									a_		:		Dict				    =				config.get(snake_case__  )
									if ds_val is not None and ds_val != hf_val:
												self.mismatches.append(F'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''  )

						_A  :							Optional[int]   			=	partialmethod(a_,       must_match=a_    )





						def 	UpperCamelCase__			(  self       ,     _lowercase  )      ->				Tuple:
									a_		:		Dict				    =				args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
									self.fill_match(
									    """train_micro_batch_size_per_gpu"""       ,     args.per_device_train_batch_size       ,     """per_device_train_batch_size"""  )
									self.fill_match("""gradient_accumulation_steps"""       ,     args.gradient_accumulation_steps       ,     """gradient_accumulation_steps"""  )
									self.fill_match("""train_batch_size"""       ,     snake_case__       ,     """train_batch_size (calculated)"""  )
									self.fill_match("""gradient_clipping"""       ,     args.max_grad_norm       ,     """max_grad_norm"""  )

									self.fill_match("""optimizer.params.lr"""       ,     args.learning_rate       ,     """learning_rate"""  )
									self.fill_match("""optimizer.params.betas"""       ,     [args.adam_betaa, args.adam_betaa]       ,     """adam_beta1+adam_beta2"""  )
									self.fill_match("""optimizer.params.eps"""       ,     args.adam_epsilon       ,     """adam_epsilon"""  )
									self.fill_match("""optimizer.params.weight_decay"""       ,     args.weight_decay       ,     """weight_decay"""  )

									self.fill_only("""scheduler.params.warmup_min_lr"""       ,     0  )  # not a trainer arg
									self.fill_match("""scheduler.params.warmup_max_lr"""       ,     args.learning_rate       ,     """learning_rate"""  )
									# total_num_steps - will get set in trainer_config_finalize

									# fp16
									if args.fpaa or args.fpaa_full_eval:
												a_		:		List[str]				    =				"""apex""" if args.fpaa_backend == """apex""" else """amp"""
									else:
												a_		:		Any				    =				None

									if args.save_on_each_node:
												# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
												a_		:		List[Any]				    =				self.config.get("""checkpoint"""       ,     {}  )
												a_		:		Union[str, Any]				    =				args.save_on_each_node

									# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
									# any here unless the user did the work
									self.fill_match(
									    """fp16.enabled"""       ,     ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == """amp""")       ,     """fp16|fp16_full_eval+fp16_backend(amp)"""       ,     )

									# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
									# ZeRO features
									self.fill_match("""amp.enabled"""       ,     fpaa_backend == """apex"""       ,     """fp16+fp16_backend(apex)"""  )
									self.fill_match("""amp.opt_level"""       ,     args.fpaa_opt_level       ,     """fp16_opt_level"""  )

									self.fill_match("""bf16.enabled"""       ,     (args.bfaa or args.bfaa_full_eval)       ,     """bf16|bf16_full_eval"""  )

									# deepspeed's default mode is fp16 unless there is a config that says differently
									if self.is_true("""bf16.enabled"""  ):
												a_		:		Optional[int]				    =				torch.bfloataa
									elif self.is_false("""fp16.enabled"""  ):
												a_		:		str				    =				torch.floataa
									else:
												a_		:		Tuple				    =				torch.floataa





						def 	UpperCamelCase__			(  self       ,     _lowercase       ,     _lowercase       ,     _lowercase  )      ->				Dict:
									a_		:		Optional[int]				    =				[
									    """zero_optimization.reduce_bucket_size""",
									    """zero_optimization.stage3_prefetch_bucket_size""",
									    """zero_optimization.stage3_param_persistence_threshold""",
									]
									a_		:		List[Any]				    =				[x for x in hidden_size_based_keys if self.is_auto(snake_case__  )]

									if len(snake_case__  ) > 0:
												if hasattr(model.config       ,     """hidden_size"""  ):
															a_		:		List[str]				    =				model.config.hidden_size
												elif hasattr(model.config       ,     """hidden_sizes"""  ):
															# if there are many hidden sizes pick the largest one
															a_		:		Union[str, Any]				    =				max(model.config.hidden_sizes  )
												else:
															raise ValueError(
															    """The model's config file has neither `hidden_size` nor `hidden_sizes` entry, """
															    """therefore it's not possible to automatically fill out the following `auto` entries """
															    F'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
															    """`auto` values for these keys with an integer value of your choice."""  )

												self.fill_only("""zero_optimization.reduce_bucket_size"""       ,     hidden_size * hidden_size  )
												if self.is_zeroa():
															# automatically assign the optimal config values based on model config
															self.fill_only("""zero_optimization.stage3_prefetch_bucket_size"""       ,     0.9 * hidden_size * hidden_size  )
															self.fill_only("""zero_optimization.stage3_param_persistence_threshold"""       ,     10 * hidden_size  )

        # scheduler
									self.fill_match("""scheduler.params.total_num_steps"""       ,     snake_case__       ,     """num_training_steps (calculated)"""  )
									self.fill_match("""scheduler.params.warmup_num_steps"""       ,     args.get_warmup_steps(snake_case__  )       ,     """warmup_steps"""  )

									if len(self.mismatches  ) > 0:
												a_		:		Optional[int]				    =				"""\n""".join(self.mismatches  )
												raise ValueError(
												    """Please correct the following DeepSpeed config values that mismatch TrainingArguments"""
												    F''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''  )







__snake_case							:     Optional[Any] 						=	None


def 			_UpperCAmelCase							(		a__) ->    Any:

			'''simple docstring'''



			global _hf_deepspeed_config_weak_ref
			# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
			a_		:		str				    =				weakref.ref(_lowerCAmelCase)


def 			_UpperCAmelCase							(		) ->    List[str]:

			'''simple docstring'''



			global _hf_deepspeed_config_weak_ref
			a_		:		List[Any]				    =				None


def 			_UpperCAmelCase							(		) ->    Dict:

			'''simple docstring'''



			if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
						return _hf_deepspeed_config_weak_ref().is_zeroa()
			else:
						return False


def 			_UpperCAmelCase							(		) ->    List[Any]:

			'''simple docstring'''



			if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
						return _hf_deepspeed_config_weak_ref().config
			else:
						return None


def 			_UpperCAmelCase							(		a__       ,					a__       ,					a__       ,					a__       ,					a__) ->    Any:

			'''simple docstring'''



			from accelerate.utils import DummyOptim, DummyScheduler

			a_		:		str				    =				hf_deepspeed_config.config

			# Optimizer + Scheduler
			# Currently supported combos:
			# 1. DS scheduler + DS optimizer: Yes
			# 2. HF scheduler + HF optimizer: Yes
			# 3. DS scheduler + HF optimizer: Yes
			# 4. HF scheduler + DS optimizer: No
			#
			# Unless Offload is enabled in which case it's:
			# 1. DS scheduler + DS optimizer: Yes
			# 2. HF scheduler + HF optimizer: Mostly*
			# 3. DS scheduler + HF optimizer: Mostly*
			# 4. HF scheduler + DS optimizer: No
			#
			# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

			a_		:		str				    =				None
			if "optimizer" in config:
						if args.adafactor:
									raise ValueError(
									    """--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. """
									    """Only one optimizer can be configured.""")
						a_		:		Any				    =				DummyOptim(params=_lowerCAmelCase)
			else:
						if hf_deepspeed_config.is_offload():
									logger.info(
									    """Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"""
									    """ custom optimizer has both CPU and GPU implementation (except LAMB)""")

						# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
						# But trainer uses AdamW by default.
						a_		:		Tuple				    =				trainer.create_optimizer()
						# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
						a_		:		Any				    =				True

			a_		:		Union[str, Any]				    =				None
			if "scheduler" in config:
						a_		:		List[str]				    =				DummyScheduler(_lowerCAmelCase)
			else:
						if isinstance(_lowerCAmelCase       ,					_lowerCAmelCase):
									raise ValueError(
									    """Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. """
									    """Please configure a scheduler in the DeepSpeed config.""")
						a_		:		int				    =				trainer.create_scheduler(num_training_steps=_lowerCAmelCase       ,					optimizer=_lowerCAmelCase)

			return optimizer, lr_scheduler


def 			_UpperCAmelCase							(		a__       ,					a__       ,					a__=False) ->    List[str]:

			'''simple docstring'''



			from deepspeed.utils import logger as ds_logger

			a_		:		Optional[Any]				    =				trainer.model
			a_		:		Optional[Any]				    =				trainer.args

			a_		:		Optional[Any]				    =				trainer.accelerator.state.deepspeed_plugin.hf_ds_config

			# resume config update - some bits like `model` and `num_training_steps` only become available during train
			hf_deepspeed_config.trainer_config_finalize(_lowerCAmelCase       ,					_lowerCAmelCase       ,					_lowerCAmelCase)

			# set the Deepspeed log level consistent with the Trainer
			ds_logger.setLevel(args.get_process_log_level())

			if inference:
						# only Z3 makes sense for the inference
						if not hf_deepspeed_config.is_zeroa():
									raise ValueError("""ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config""")

						# in case the training config is re-used for inference
						hf_deepspeed_config.del_config_sub_tree("""optimizer""")
						hf_deepspeed_config.del_config_sub_tree("""lr_scheduler""")
						a_    ,			a_		:		str				    =				None, None
						a_		:		Dict				    =				None
			else:
						a_		:		Dict				    =				None  # important for when deepspeed_init is used as re-init
						a_		:		int				    =				list(filter(lambda a__: p.requires_grad       ,					model.parameters()))
						a_    ,			a_		:		int				    =				deepspeed_optim_sched(
						    _lowerCAmelCase       ,					_lowerCAmelCase       ,					_lowerCAmelCase       ,					_lowerCAmelCase       ,					_lowerCAmelCase)

			# keep for quick debug:
			# from pprint import pprint; pprint(config)

			return optimizer, lr_scheduler



def 			_UpperCAmelCase							(		a__       ,					a__) ->    Any:

			'''simple docstring'''



			import glob

			a_		:		int				    =				sorted(glob.glob(f'''{checkpoint_path}/global_step*'''))

			if len(_lowerCAmelCase) > 0:
						logger.info(f'''Attempting to resume from {checkpoint_path}''')
						# this magically updates self.optimizer and self.lr_scheduler
						a_    ,			a_		:		List[str]				    =				deepspeed_engine.load_checkpoint(
						    _lowerCAmelCase       ,					load_optimizer_states=_lowerCAmelCase       ,					load_lr_scheduler_states=_lowerCAmelCase)
						if load_path is None:
									raise ValueError(f'''[deepspeed] failed to resume from checkpoint {checkpoint_path}''')
			else:
						raise ValueError(f'''Can\'t find a valid checkpoint at {checkpoint_path}''')





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							:     Optional[Any] 						=	logging.get_logger(__name__)

__snake_case							:     str 						=	{
    """ZinengTang/tvlt-base""": """https://huggingface.co/ZinengTang/tvlt-base/blob/main/config.json""",
}



class    A__(a_    ):
						"""simple docstring"""






						_A  :							Any   			=	'''tvlt'''





						def __init__(  self       ,     _lowercase=224       ,     _lowercase=2_048       ,     _lowercase=128       ,     _lowercase=[16, 16]       ,     _lowercase=[16, 16]       ,     _lowercase=3       ,     _lowercase=1       ,     _lowercase=8       ,     _lowercase=768       ,     _lowercase=12       ,     _lowercase=12       ,     _lowercase=3_072       ,     _lowercase="gelu"       ,     _lowercase=0.0       ,     _lowercase=0.0       ,     _lowercase=0.0_2       ,     _lowercase=1e-6       ,     _lowercase=True       ,     _lowercase=False       ,     _lowercase=16       ,     _lowercase=512       ,     _lowercase=8       ,     _lowercase=2_048       ,     _lowercase=0.7_5       ,     _lowercase=0.1_5       ,     _lowercase="frame-level"       ,     _lowercase=True       ,     _lowercase=True       ,     _lowercase="classification"       ,     **_lowercase       ,     )      ->				Tuple:
									super().__init__(**_lowercase  )

									if audio_mask_type not in ("frame-level", "patch_level"):
												raise ValueError(
												    """audio_mask_type must be one of two acceptable strategies - {'frame_level', 'patch-level') """
												    F'''got {audio_mask_type}'''  )

									a_		:		int				    =				image_size
									a_		:		str				    =				spectrogram_length
									a_		:		int				    =				frequency_length
									a_		:		str				    =				image_patch_size
									a_		:		int				    =				audio_patch_size
									a_		:		Any				    =				num_image_channels
									a_		:		str				    =				num_audio_channels
									a_		:		int				    =				num_frames

									a_		:		Optional[int]				    =				hidden_size
									a_		:		Tuple				    =				num_hidden_layers
									a_		:		Any				    =				num_attention_heads
									a_		:		Union[str, Any]				    =				intermediate_size
									a_		:		Tuple				    =				hidden_act
									a_		:		Union[str, Any]				    =				hidden_dropout_prob
									a_		:		int				    =				attention_probs_dropout_prob
									a_		:		Any				    =				initializer_range
									a_		:		int				    =				layer_norm_eps
									a_		:		Union[str, Any]				    =				qkv_bias
									a_		:		str				    =				use_mean_pooling

									a_		:		List[Any]				    =				decoder_num_attention_heads
									a_		:		Optional[Any]				    =				decoder_hidden_size
									a_		:		Dict				    =				decoder_num_hidden_layers
									a_		:		Optional[Any]				    =				decoder_intermediate_size
									a_		:		Dict				    =				pixel_mask_ratio
									a_		:		int				    =				audio_mask_ratio
									a_		:		Optional[Any]				    =				audio_mask_type

									a_		:		Union[str, Any]				    =				task_matching
									a_		:		Dict				    =				task_mae
									a_		:		Union[str, Any]				    =				loss_type


