






import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase     )->  Optional[int]:





						"""simple docstring"""



						if isinstance(snake_case__			,		(list, tuple)     ):
												return honor_type(snake_case__			,		(move_to_device(snake_case__			,		snake_case__     ) for t in state)     )
						elif isinstance(snake_case__			,		snake_case__     ):
												return type(snake_case__     )({k: move_to_device(snake_case__			,		snake_case__     ) for k, v in state.items()}     )
						elif isinstance(snake_case__			,		torch.Tensor     ):
												return state.to(snake_case__     )
						return state





class __lowerCamelCase			(				torch.optim.Optimizer):





	"""simple docstring"""






	def __init__(							self	,   UpperCAmelCase	,   UpperCAmelCase=True	,   UpperCAmelCase=None   ):
							"""simple docstring"""







							_UpperCAmelCase     =      optimizer
							_UpperCAmelCase     =      scaler
							_UpperCAmelCase     =      AcceleratorState()
							_UpperCAmelCase     =      GradientState()
							_UpperCAmelCase     =      device_placement
							_UpperCAmelCase     =      False
							_UpperCAmelCase     =      None

							# Handle device placement
							if device_placement:
													_UpperCAmelCase     =      self.optimizer.state_dict()
													if self.accelerator_state.distributed_type == DistributedType.TPU:
																			xm.send_cpu_data_to_device(UpperCAmelCase	,   self.accelerator_state.device   )
													else:
																			_UpperCAmelCase     =      move_to_device(UpperCAmelCase	,   self.accelerator_state.device   )
													self.optimizer.load_state_dict(UpperCAmelCase   )

	@property
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return self.optimizer.state

	@state.setter
	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      state

	@property
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return self.optimizer.param_groups

	@param_groups.setter
	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      param_groups

	@property
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return self.optimizer.defaults

	@defaults.setter
	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      defaults

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							self.optimizer.add_param_group(UpperCAmelCase   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
													xm.send_cpu_data_to_device(UpperCAmelCase	,   self.accelerator_state.device   )
							self.optimizer.load_state_dict(UpperCAmelCase   )

	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return self.optimizer.state_dict()

	def 	UpperCamelCase   (							self	,   UpperCAmelCase=None   ):
							"""simple docstring"""







							if self.gradient_state.sync_gradients:
													_UpperCAmelCase     =      'set_to_none' in inspect.signature(self.optimizer.zero_grad   ).parameters
													if accept_arg:
																			if set_to_none is None:
																									_UpperCAmelCase     =      False
																			self.optimizer.zero_grad(set_to_none=UpperCAmelCase   )
													else:
																			if set_to_none is not None:
																									raise ValueError('`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.'   )
																			self.optimizer.zero_grad()

	def 	UpperCamelCase   (							self	,   UpperCAmelCase=None   ):
							"""simple docstring"""







							if self.gradient_state.sync_gradients:
													if self.accelerator_state.distributed_type == DistributedType.TPU:
																			_UpperCAmelCase     =      {'closure': closure} if closure is not None else {}
																			xm.optimizer_step(self.optimizer	,   optimizer_args=UpperCAmelCase   )
													elif self.scaler is not None:
																			_UpperCAmelCase     =      False
																			if self._last_scale is None:
																									# `get_scale` is an async operation requiring full synchronization
																									# on CPU and GPUs before finishing. As a result, we store away
																									# the prior one to reduce the call overhead
																									_UpperCAmelCase     =      self.scaler.get_scale()
																									_UpperCAmelCase     =      True
																			self.scaler.step(self.optimizer	,   UpperCAmelCase   )
																			self.scaler.update()
																			_UpperCAmelCase     =      self.scaler.get_scale()
																			if not new_scale:
																									# If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
																									_UpperCAmelCase     =      scale_after < self._last_scale
																			_UpperCAmelCase     =      scale_after

													else:
																			self.optimizer.step(UpperCAmelCase   )

	def 	UpperCamelCase   (							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							for param_group in self.optimizer.param_groups:
													_UpperCAmelCase     =      [parameters_map.get(UpperCAmelCase	,   UpperCAmelCase   ) for p in param_group['params']]

	@property
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							warnings.warn(
							    'The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use '
							    '`optimizer.step_was_skipped` instead.'	,   UpperCAmelCase	,   )
							return self._is_overflow

	@property
	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return self._is_overflow

	def __getstate__(							self   ):
							"""simple docstring"""







							return self.__dict__.copy()

	def __setstate__(							self	,   UpperCAmelCase   ):
							"""simple docstring"""







							self.__dict__.update(UpperCAmelCase   )








from ..utils import DummyObject, requires_backends





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )





class __lowerCamelCase			(				metaclass=snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=				["tokenizers"]

	def __init__(							self	,   *UpperCAmelCase	,   **UpperCAmelCase   ):
							"""simple docstring"""







							requires_backends(self	,   ['tokenizers']   )
