




'''simple docstring'''






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
				import torch

_lowerCAmelCase				     = logging.get_logger(__name__)

def       __lowerCAmelCase (   ):
			return importlib.util.find_spec("deepspeed"						) is not None


if is_accelerate_available() and is_deepspeed_available():
				from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
				# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
				# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
				from builtins import object as DeepSpeedConfig







class  A		(			_A		):




				'''simple docstring'''





				def __init__(self			,  _UpperCAmelCase      )     ->	Dict:
							set_hf_deepspeed_config(self      )
							dep_version_check("accelerate"      )
							dep_version_check("deepspeed"      )
							super().__init__(UpperCamelCase__      )







class  A		(			_A		):




				'''simple docstring'''





				def __init__(self			,  _UpperCAmelCase      )     ->	Any:
							super().__init__(UpperCamelCase__      )
							__UpperCamelCase				:      Tuple									=       None
							__UpperCamelCase				:      str									=       []
				def 	a_	(self      )     ->	Union[str, Any]:






							if self._dtype is None:
										raise ValueError("trainer_config_process() wasn't called yet to tell dtype"      )
							return self._dtype
				def 	a_	(self			,  _UpperCAmelCase      )     ->	List[Any]:
							__UpperCamelCase				:      int									=       self.get_value(UpperCamelCase__      )
							if val is None:
										return False
							else:
										return val == "auto"
				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase=None			,  _UpperCAmelCase=True      )     ->	Dict:
							__UpperCamelCase		, __UpperCamelCase				:      List[str]									=       self.find_config_node(UpperCamelCase__      )
							if config is None:
										return

							if config.get(UpperCamelCase__      ) == "auto":
										__UpperCamelCase				:      int									=       hf_val
										return

							if not must_match:
										return

							__UpperCamelCase				:      Any									=       config.get(UpperCamelCase__      )
							if ds_val is not None and ds_val != hf_val:
										self.mismatches.append(f"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"      )

				A				     =    partialmethod(_A      ,							must_match=_A		)
				def 	a_	(self			,  _UpperCAmelCase      )     ->	List[Any]:
							__UpperCamelCase				:      Union[str, Any]									=       args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
							self.fill_match(
							    "train_micro_batch_size_per_gpu"			,  args.per_device_train_batch_size			,  "per_device_train_batch_size"      )
							self.fill_match("gradient_accumulation_steps"			,  args.gradient_accumulation_steps			,  "gradient_accumulation_steps"      )
							self.fill_match("train_batch_size"			,  UpperCamelCase__			,  "train_batch_size (calculated)"      )
							self.fill_match("gradient_clipping"			,  args.max_grad_norm			,  "max_grad_norm"      )

							self.fill_match("optimizer.params.lr"			,  args.learning_rate			,  "learning_rate"      )
							self.fill_match("optimizer.params.betas"			,  [args.adam_betaa, args.adam_betaa]			,  "adam_beta1+adam_beta2"      )
							self.fill_match("optimizer.params.eps"			,  args.adam_epsilon			,  "adam_epsilon"      )
							self.fill_match("optimizer.params.weight_decay"			,  args.weight_decay			,  "weight_decay"      )

							self.fill_only("scheduler.params.warmup_min_lr"			,  0      )  # not a trainer arg
							self.fill_match("scheduler.params.warmup_max_lr"			,  args.learning_rate			,  "learning_rate"      )
							# total_num_steps - will get set in trainer_config_finalize

							# fp16
							if args.fpaa or args.fpaa_full_eval:
										__UpperCamelCase				:      Any									=       "apex" if args.fpaa_backend == "apex" else "amp"
							else:
										__UpperCamelCase				:      Any									=       None

							if args.save_on_each_node:
										# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
										__UpperCamelCase				:      Dict									=       self.config.get("checkpoint"			,  {}      )
										__UpperCamelCase				:      Dict									=       args.save_on_each_node

							# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
							# any here unless the user did the work
							self.fill_match(
							    "fp16.enabled"			,  ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")			,  "fp16|fp16_full_eval+fp16_backend(amp)"			,  )

							# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
							# ZeRO features
							self.fill_match("amp.enabled"			,  fpaa_backend == "apex"			,  "fp16+fp16_backend(apex)"      )
							self.fill_match("amp.opt_level"			,  args.fpaa_opt_level			,  "fp16_opt_level"      )

							self.fill_match("bf16.enabled"			,  (args.bfaa or args.bfaa_full_eval)			,  "bf16|bf16_full_eval"      )

							# deepspeed's default mode is fp16 unless there is a config that says differently
							if self.is_true("bf16.enabled"      ):
										__UpperCamelCase				:      Tuple									=       torch.bfloataa
							elif self.is_false("fp16.enabled"      ):
										__UpperCamelCase				:      Any									=       torch.floataa
							else:
										__UpperCamelCase				:      Optional[int]									=       torch.floataa





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase      )     ->	Dict:
							__UpperCamelCase				:      List[str]									=       [
							    "zero_optimization.reduce_bucket_size",
							    "zero_optimization.stage3_prefetch_bucket_size",
							    "zero_optimization.stage3_param_persistence_threshold",
							]
							__UpperCamelCase				:      List[str]									=       [x for x in hidden_size_based_keys if self.is_auto(UpperCamelCase__      )]

							if len(UpperCamelCase__      ) > 0:
										if hasattr(model.config			,  "hidden_size"      ):
													__UpperCamelCase				:      Union[str, Any]									=       model.config.hidden_size
										elif hasattr(model.config			,  "hidden_sizes"      ):
													# if there are many hidden sizes pick the largest one
													__UpperCamelCase				:      Dict									=       max(model.config.hidden_sizes      )
										else:
													raise ValueError(
													    "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, "
													    "therefore it's not possible to automatically fill out the following `auto` entries "
													    f"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing "
													    "`auto` values for these keys with an integer value of your choice."      )

										self.fill_only("zero_optimization.reduce_bucket_size"			,  hidden_size * hidden_size      )
										if self.is_zeroa():
													# automatically assign the optimal config values based on model config
													self.fill_only("zero_optimization.stage3_prefetch_bucket_size"			,  0.9 * hidden_size * hidden_size      )
													self.fill_only("zero_optimization.stage3_param_persistence_threshold"			,  1_0 * hidden_size      )

													# scheduler
							self.fill_match("scheduler.params.total_num_steps"			,  UpperCamelCase__			,  "num_training_steps (calculated)"      )
							self.fill_match("scheduler.params.warmup_num_steps"			,  args.get_warmup_steps(UpperCamelCase__      )			,  "warmup_steps"      )

							if len(self.mismatches      ) > 0:
										__UpperCamelCase				:      List[str]									=       "\n".join(self.mismatches      )
										raise ValueError(
										    "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
										    f" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'."      )







_lowerCAmelCase				     = None

def       __lowerCAmelCase (   snake_case__						):
			global _hf_deepspeed_config_weak_ref
			# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
			__UpperCamelCase				:      Any									=       weakref.ref(A_						)

def       __lowerCAmelCase (   ):
			global _hf_deepspeed_config_weak_ref
			__UpperCamelCase				:      str									=       None

def       __lowerCAmelCase (   ):





			if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
						return _hf_deepspeed_config_weak_ref().is_zeroa()
			else:
						return False

def       __lowerCAmelCase (   ):





			if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
						return _hf_deepspeed_config_weak_ref().config
			else:
						return None

def       __lowerCAmelCase (   snake_case__ ,				snake_case__ ,				snake_case__ ,				snake_case__ ,				snake_case__						):
			from accelerate.utils import DummyOptim, DummyScheduler

			__UpperCamelCase				:      Optional[Any]									=       hf_deepspeed_config.config

			# Optimizer + Scheduler
			# Currently supported combos:
			# 1. DS scheduler + DS optimizer: Yes
			# 2. HF scheduler + HF optimizer: Yes
			# 3. DS scheduler + HF optimizer: Yes
			# 4. HF scheduler + DS optimizer: No
			#
			# Unless Offload is enabled in which case it's:
			# 1. DS scheduler + DS optimizer: Yes
			# 2. HF scheduler + HF optimizer: Mostly*
			# 3. DS scheduler + HF optimizer: Mostly*
			# 4. HF scheduler + DS optimizer: No
			#
			# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

			__UpperCamelCase				:      Any									=       None
			if "optimizer" in config:
						if args.adafactor:
									raise ValueError(
									    "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
									    "Only one optimizer can be configured."						)
						__UpperCamelCase				:      Dict									=       DummyOptim(params=A_						)
			else:
						if hf_deepspeed_config.is_offload():
									logger.info(
									    "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
									    " custom optimizer has both CPU and GPU implementation (except LAMB)"						)

						# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
						# But trainer uses AdamW by default.
						__UpperCamelCase				:      Any									=       trainer.create_optimizer()
						# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
						__UpperCamelCase				:      Tuple									=       True

			__UpperCamelCase				:      Any									=       None
			if "scheduler" in config:
						__UpperCamelCase				:      Optional[int]									=       DummyScheduler(A_						)
			else:
						if isinstance(A_ ,				A_						):
									raise ValueError(
									    "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
									    "Please configure a scheduler in the DeepSpeed config."						)
						__UpperCamelCase				:      Optional[Any]									=       trainer.create_scheduler(num_training_steps=A_ ,				optimizer=A_						)

			return optimizer, lr_scheduler

def       __lowerCAmelCase (   snake_case__ ,				snake_case__ ,				snake_case__=False						):
			from deepspeed.utils import logger as ds_logger

			__UpperCamelCase				:      Tuple									=       trainer.model
			__UpperCamelCase				:      Dict									=       trainer.args

			__UpperCamelCase				:      Tuple									=       trainer.accelerator.state.deepspeed_plugin.hf_ds_config

			# resume config update - some bits like `model` and `num_training_steps` only become available during train
			hf_deepspeed_config.trainer_config_finalize(A_ ,				A_ ,				A_						)

			# set the Deepspeed log level consistent with the Trainer
			ds_logger.setLevel(args.get_process_log_level()						)

			if inference:
						# only Z3 makes sense for the inference
						if not hf_deepspeed_config.is_zeroa():
									raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"						)

						# in case the training config is re-used for inference
						hf_deepspeed_config.del_config_sub_tree("optimizer"						)
						hf_deepspeed_config.del_config_sub_tree("lr_scheduler"						)
						__UpperCamelCase		, __UpperCamelCase				:      Dict									=       None, None
						__UpperCamelCase				:      Any									=       None
			else:
						__UpperCamelCase				:      Optional[Any]									=       None  # important for when deepspeed_init is used as re-init
						__UpperCamelCase				:      Optional[int]									=       list(filter(lambda snake_case__						: p.requires_grad ,				model.parameters()						)						)
						__UpperCamelCase		, __UpperCamelCase				:      Optional[int]									=       deepspeed_optim_sched(
						    A_ ,				A_ ,				A_ ,				A_ ,				A_						)

			# keep for quick debug:
			# from pprint import pprint; pprint(config)

			return optimizer, lr_scheduler




def       __lowerCAmelCase (   snake_case__ ,				snake_case__						):
			import glob

			__UpperCamelCase				:      str									=       sorted(glob.glob(F"{checkpoint_path}/global_step*"						)						)

			if len(A_						) > 0:
						logger.info(F"Attempting to resume from {checkpoint_path}"						)
						# this magically updates self.optimizer and self.lr_scheduler
						__UpperCamelCase		, __UpperCamelCase				:      List[Any]									=       deepspeed_engine.load_checkpoint(
						    A_ ,				load_optimizer_states=A_ ,				load_lr_scheduler_states=A_						)
						if load_path is None:
									raise ValueError(F"[deepspeed] failed to resume from checkpoint {checkpoint_path}"						)
			else:
						raise ValueError(F"Can\'t find a valid checkpoint at {checkpoint_path}"						)









'''simple docstring'''






from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


_lowerCAmelCase				     = logging.get_logger(__name__)

_lowerCAmelCase				     = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCAmelCase				     = {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
    '''tokenizer_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/tokenizer.json'''},
}

_lowerCAmelCase				     = {
    '''openai-gpt''': 512,
}



class  A		(			SCREAMING_SNAKE_CASE__		):




				'''simple docstring'''





				A				     =    VOCAB_FILES_NAMES
				A				     =    PRETRAINED_VOCAB_FILES_MAP
				A				     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A				     =    ["input_ids", "attention_mask"]
				A				     =    OpenAIGPTTokenizer
				def __init__(self			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase=None			,  _UpperCAmelCase="<unk>"			,  **_UpperCAmelCase      )     ->	str:
							super().__init__(_UpperCAmelCase			,  _UpperCAmelCase			,  tokenizer_file=_UpperCAmelCase			,  unk_token=_UpperCAmelCase			,  **_UpperCAmelCase      )
				@property
				def 	a_	(self      )     ->	Dict:
							return True





				def 	a_	(self			,  _UpperCAmelCase			,  _UpperCAmelCase = None      )     ->	Tuple[str]:
							__UpperCamelCase				:      Optional[Any]									=       self._tokenizer.model.save(_UpperCAmelCase			,  name=_UpperCAmelCase      )
							return tuple(_UpperCAmelCase      )



