





import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
   import torch

A__:  List[Any]       					=					logging.get_logger(__name__)



def 							lowerCAmelCase_				(    ):
    return importlib.util.find_spec("deepspeed") is not None


if is_accelerate_available() and is_deepspeed_available():
   from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
   # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
   # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
   from builtins import object as DeepSpeedConfig




class    _a  (  _A):

  """simple docstring"""


  def __init__(						self:	Tuple					,  __lowerCamelCase:	Optional[int]			):
      '''simple docstring'''


      set_hf_deepspeed_config(self			)
      dep_version_check("accelerate"			)
      dep_version_check("deepspeed"			)
      super().__init__(UpperCamelCase__			)




class    _a  (  _A):

  """simple docstring"""


  def __init__(						self:	List[Any]					,  __lowerCamelCase:	Dict			):
      '''simple docstring'''


      super().__init__(UpperCamelCase__			)
      UpperCamelCase__:					str												=      None
      UpperCamelCase__:					Dict												=      []






  def 	UpperCAmelCase_   (						self:	Optional[Any]			):
      '''simple docstring'''


      if self._dtype is None:
          raise ValueError("trainer_config_process() wasn't called yet to tell dtype"			)
      return self._dtype






  def 	UpperCAmelCase_   (						self:	Any					,  __lowerCamelCase:	Any			):
      '''simple docstring'''


      UpperCamelCase__:					Dict												=      self.get_value(UpperCamelCase__			)
      if val is None:
          return False
      else:
          return val == "auto"






  def 	UpperCAmelCase_   (						self:	Optional[int]					,  __lowerCamelCase:	List[Any]					,  __lowerCamelCase:	List[str]					,  __lowerCamelCase:	Optional[Any]=None					,  __lowerCamelCase:	Tuple=True			):
      '''simple docstring'''


      UpperCamelCase__							,				UpperCamelCase__:					List[str]												=      self.find_config_node(UpperCamelCase__			)
      if config is None:
          return

      if config.get(UpperCamelCase__			) == "auto":
          UpperCamelCase__:					List[Any]												=      hf_val
          return

      if not must_match:
          return

      UpperCamelCase__:					Optional[Any]												=      config.get(UpperCamelCase__			)
      if ds_val is not None and ds_val != hf_val:
          self.mismatches.append(F"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"			)

  UpperCamelCase__       						=   partialmethod(_A		,	must_match=_A)






  def 	UpperCAmelCase_   (						self:	Any					,  __lowerCamelCase:	Optional[Any]			):
      '''simple docstring'''


      UpperCamelCase__:					int												=      args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
      self.fill_match(
          "train_micro_batch_size_per_gpu"					,  args.per_device_train_batch_size					,  "per_device_train_batch_size"			)
      self.fill_match("gradient_accumulation_steps"					,  args.gradient_accumulation_steps					,  "gradient_accumulation_steps"			)
      self.fill_match("train_batch_size"					,  UpperCamelCase__					,  "train_batch_size (calculated)"			)
      self.fill_match("gradient_clipping"					,  args.max_grad_norm					,  "max_grad_norm"			)

      self.fill_match("optimizer.params.lr"					,  args.learning_rate					,  "learning_rate"			)
      self.fill_match("optimizer.params.betas"					,  [args.adam_betaa, args.adam_betaa]					,  "adam_beta1+adam_beta2"			)
      self.fill_match("optimizer.params.eps"					,  args.adam_epsilon					,  "adam_epsilon"			)
      self.fill_match("optimizer.params.weight_decay"					,  args.weight_decay					,  "weight_decay"			)

      self.fill_only("scheduler.params.warmup_min_lr"					,  0			)  # not a trainer arg
      self.fill_match("scheduler.params.warmup_max_lr"					,  args.learning_rate					,  "learning_rate"			)
      # total_num_steps - will get set in trainer_config_finalize

      # fp16
      if args.fpaa or args.fpaa_full_eval:
          UpperCamelCase__:					str												=      "apex" if args.fpaa_backend == "apex" else "amp"
      else:
          UpperCamelCase__:					List[Any]												=      None

      if args.save_on_each_node:
          # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
          UpperCamelCase__:					str												=      self.config.get("checkpoint"					,  {}			)
          UpperCamelCase__:					Tuple												=      args.save_on_each_node

      # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
      # any here unless the user did the work
      self.fill_match(
          "fp16.enabled"					,  ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")					,  "fp16|fp16_full_eval+fp16_backend(amp)"					,  )

      # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
      # ZeRO features
      self.fill_match("amp.enabled"					,  fpaa_backend == "apex"					,  "fp16+fp16_backend(apex)"			)
      self.fill_match("amp.opt_level"					,  args.fpaa_opt_level					,  "fp16_opt_level"			)

      self.fill_match("bf16.enabled"					,  (args.bfaa or args.bfaa_full_eval)					,  "bf16|bf16_full_eval"			)

      # deepspeed's default mode is fp16 unless there is a config that says differently
      if self.is_true("bf16.enabled"			):
          UpperCamelCase__:					Union[str, Any]												=      torch.bfloataa
      elif self.is_false("fp16.enabled"			):
          UpperCamelCase__:					Optional[int]												=      torch.floataa
      else:
          UpperCamelCase__:					int												=      torch.floataa






  def 	UpperCAmelCase_   (						self:	List[str]					,  __lowerCamelCase:	List[str]					,  __lowerCamelCase:	Union[str, Any]					,  __lowerCamelCase:	Tuple			):
      '''simple docstring'''


      UpperCamelCase__:					Tuple												=      [
          "zero_optimization.reduce_bucket_size",
          "zero_optimization.stage3_prefetch_bucket_size",
          "zero_optimization.stage3_param_persistence_threshold",
      ]
      UpperCamelCase__:					Optional[int]												=      [x for x in hidden_size_based_keys if self.is_auto(UpperCamelCase__			)]

      if len(UpperCamelCase__			) > 0:
          if hasattr(model.config					,  "hidden_size"			):
              UpperCamelCase__:					Tuple												=      model.config.hidden_size
          elif hasattr(model.config					,  "hidden_sizes"			):
              # if there are many hidden sizes pick the largest one
              UpperCamelCase__:					Optional[int]												=      max(model.config.hidden_sizes			)
          else:
              raise ValueError(
                  "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, "
                  "therefore it's not possible to automatically fill out the following `auto` entries "
                  F"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing "
                  "`auto` values for these keys with an integer value of your choice."			)

          self.fill_only("zero_optimization.reduce_bucket_size"					,  hidden_size * hidden_size			)
          if self.is_zeroa():
              # automatically assign the optimal config values based on model config
              self.fill_only("zero_optimization.stage3_prefetch_bucket_size"					,  0.9 * hidden_size * hidden_size			)
              self.fill_only("zero_optimization.stage3_param_persistence_threshold"					,  10 * hidden_size			)

              # scheduler
      self.fill_match("scheduler.params.total_num_steps"					,  UpperCamelCase__					,  "num_training_steps (calculated)"			)
      self.fill_match("scheduler.params.warmup_num_steps"					,  args.get_warmup_steps(UpperCamelCase__			)					,  "warmup_steps"			)

      if len(self.mismatches			) > 0:
          UpperCamelCase__:					Union[str, Any]												=      "\n".join(self.mismatches			)
          raise ValueError(
              "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
              F" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'."			)
A__:  Optional[int]       					=					None



def 							lowerCAmelCase_				(    A_):
    global _hf_deepspeed_config_weak_ref
    # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
    UpperCamelCase__:					Any												=      weakref.ref(A_)



def 							lowerCAmelCase_				(    ):
    global _hf_deepspeed_config_weak_ref
    UpperCamelCase__:					str												=      None



def 							lowerCAmelCase_				(    ):





    if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
        return _hf_deepspeed_config_weak_ref().is_zeroa()
    else:
        return False



def 							lowerCAmelCase_				(    ):





    if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
        return _hf_deepspeed_config_weak_ref().config
    else:
        return None



def 							lowerCAmelCase_				(    A_  ,A_  ,A_  ,A_  ,A_):
    from accelerate.utils import DummyOptim, DummyScheduler

    UpperCamelCase__:					Dict												=      hf_deepspeed_config.config

    # Optimizer + Scheduler
    # Currently supported combos:
    # 1. DS scheduler + DS optimizer: Yes
    # 2. HF scheduler + HF optimizer: Yes
    # 3. DS scheduler + HF optimizer: Yes
    # 4. HF scheduler + DS optimizer: No
    #
    # Unless Offload is enabled in which case it's:
    # 1. DS scheduler + DS optimizer: Yes
    # 2. HF scheduler + HF optimizer: Mostly*
    # 3. DS scheduler + HF optimizer: Mostly*
    # 4. HF scheduler + DS optimizer: No
    #
    # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

    UpperCamelCase__:					str												=      None
    if "optimizer" in config:
        if args.adafactor:
            raise ValueError(
                "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
                "Only one optimizer can be configured.")
        UpperCamelCase__:					Optional[int]												=      DummyOptim(params=A_)
    else:
        if hf_deepspeed_config.is_offload():
            logger.info(
                "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
                " custom optimizer has both CPU and GPU implementation (except LAMB)")

        # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
        # But trainer uses AdamW by default.
        UpperCamelCase__:					Any												=      trainer.create_optimizer()
        # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
        UpperCamelCase__:					List[str]												=      True

    UpperCamelCase__:					Union[str, Any]												=      None
    if "scheduler" in config:
        UpperCamelCase__:					int												=      DummyScheduler(A_)
    else:
        if isinstance(A_  ,A_):
            raise ValueError(
                "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
                "Please configure a scheduler in the DeepSpeed config.")
        UpperCamelCase__:					Dict												=      trainer.create_scheduler(num_training_steps=A_  ,optimizer=A_)

    return optimizer, lr_scheduler



def 							lowerCAmelCase_				(    A_  ,A_  ,A_=False):
    from deepspeed.utils import logger as ds_logger

    UpperCamelCase__:					Tuple												=      trainer.model
    UpperCamelCase__:					int												=      trainer.args

    UpperCamelCase__:					int												=      trainer.accelerator.state.deepspeed_plugin.hf_ds_config

    # resume config update - some bits like `model` and `num_training_steps` only become available during train
    hf_deepspeed_config.trainer_config_finalize(A_  ,A_  ,A_)

    # set the Deepspeed log level consistent with the Trainer
    ds_logger.setLevel(args.get_process_log_level())

    if inference:
        # only Z3 makes sense for the inference
        if not hf_deepspeed_config.is_zeroa():
            raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config")

        # in case the training config is re-used for inference
        hf_deepspeed_config.del_config_sub_tree("optimizer")
        hf_deepspeed_config.del_config_sub_tree("lr_scheduler")
        UpperCamelCase__							,				UpperCamelCase__:					Tuple												=      None, None
        UpperCamelCase__:					Tuple												=      None
    else:
        UpperCamelCase__:					Optional[int]												=      None  # important for when deepspeed_init is used as re-init
        UpperCamelCase__:					Any												=      list(filter(lambda A_: p.requires_grad  ,model.parameters()))
        UpperCamelCase__							,				UpperCamelCase__:					int												=      deepspeed_optim_sched(
            A_  ,A_  ,A_  ,A_  ,A_)

    # keep for quick debug:
    # from pprint import pprint; pprint(config)

    return optimizer, lr_scheduler




def 							lowerCAmelCase_				(    A_  ,A_):
    import glob

    UpperCamelCase__:					List[Any]												=      sorted(glob.glob(F"{checkpoint_path}/global_step*"))

    if len(A_) > 0:
        logger.info(F"Attempting to resume from {checkpoint_path}")
        # this magically updates self.optimizer and self.lr_scheduler
        UpperCamelCase__							,				UpperCamelCase__:					List[str]												=      deepspeed_engine.load_checkpoint(
            A_  ,load_optimizer_states=A_  ,load_lr_scheduler_states=A_)
        if load_path is None:
            raise ValueError(F"[deepspeed] failed to resume from checkpoint {checkpoint_path}")
    else:
        raise ValueError(F"Can\'t find a valid checkpoint at {checkpoint_path}")








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__:  List[str]       					=					{
    '''configuration_wav2vec2_conformer''': [
        '''WAV2VEC2_CONFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''Wav2Vec2ConformerConfig''',
    ],
}


try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A__:  int       					=					[
       '''WAV2VEC2_CONFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''Wav2Vec2ConformerForAudioFrameClassification''',
       '''Wav2Vec2ConformerForCTC''',
       '''Wav2Vec2ConformerForPreTraining''',
       '''Wav2Vec2ConformerForSequenceClassification''',
       '''Wav2Vec2ConformerForXVector''',
       '''Wav2Vec2ConformerModel''',
       '''Wav2Vec2ConformerPreTrainedModel''',
   ]

if TYPE_CHECKING:
   from .configuration_wavaveca_conformer import (
       WAV2VEC2_CONFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
       WavaVecaConformerConfig,
   )

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_wavaveca_conformer import (
          WAV2VEC2_CONFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          WavaVecaConformerForAudioFrameClassification,
          WavaVecaConformerForCTC,
          WavaVecaConformerForPreTraining,
          WavaVecaConformerForSequenceClassification,
          WavaVecaConformerForXVector,
          WavaVecaConformerModel,
          WavaVecaConformerPreTrainedModel,
      )

else:
   import sys

   A__:  Dict       					=					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

