



import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm





def 						a__     (						_UpperCamelCase     :	List[Any]	,_UpperCamelCase     :	Union[str, Any]							):
			if isinstance(_UpperCamelCase	,(list, tuple)							):
						return honor_type(_UpperCamelCase	,(move_to_device(_UpperCamelCase	,_UpperCamelCase							) for t in state)							)
			elif isinstance(_UpperCamelCase	,_UpperCamelCase							):
						return type(_UpperCamelCase							)({k: move_to_device(_UpperCamelCase	,_UpperCamelCase							) for k, v in state.items()}							)
			elif isinstance(_UpperCamelCase	,torch.Tensor							):
						return state.to(_UpperCamelCase							)
			return state





class 	__lowerCAmelCase       (		torch.optim.Optimizer				):




	def __init__(  self		,			__UpperCAmelCase		,			__UpperCAmelCase=True		,			__UpperCAmelCase=None			):



				'''simple docstring'''
				__lowerCamelCase  							=      optimizer
				__lowerCamelCase  							=      scaler
				__lowerCamelCase  							=      AcceleratorState()
				__lowerCamelCase  							=      GradientState()
				__lowerCamelCase  							=      device_placement
				__lowerCamelCase  							=      False
				__lowerCamelCase  							=      None

				# Handle device placement
				if device_placement:
							__lowerCamelCase  							=      self.optimizer.state_dict()
							if self.accelerator_state.distributed_type == DistributedType.TPU:
										xm.send_cpu_data_to_device(__UpperCAmelCase		,			self.accelerator_state.device			)
							else:
										__lowerCamelCase  							=      move_to_device(__UpperCAmelCase		,			self.accelerator_state.device			)
							self.optimizer.load_state_dict(__UpperCAmelCase			)




	@property
	def     lowerCamelCase (  self			):



				'''simple docstring'''
				return self.optimizer.state




	@state.setter
	def     lowerCamelCase (  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				__lowerCamelCase  							=      state




	@property
	def     lowerCamelCase (  self			):



				'''simple docstring'''
				return self.optimizer.param_groups




	@param_groups.setter
	def     lowerCamelCase (  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				__lowerCamelCase  							=      param_groups




	@property
	def     lowerCamelCase (  self			):



				'''simple docstring'''
				return self.optimizer.defaults




	@defaults.setter
	def     lowerCamelCase (  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				__lowerCamelCase  							=      defaults




	def     lowerCamelCase (  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				self.optimizer.add_param_group(__UpperCAmelCase			)




	def     lowerCamelCase (  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
							xm.send_cpu_data_to_device(__UpperCAmelCase		,			self.accelerator_state.device			)
				self.optimizer.load_state_dict(__UpperCAmelCase			)




	def     lowerCamelCase (  self			):



				'''simple docstring'''
				return self.optimizer.state_dict()




	def     lowerCamelCase (  self		,			__UpperCAmelCase=None			):



				'''simple docstring'''
				if self.gradient_state.sync_gradients:
							__lowerCamelCase  							=      '''set_to_none''' in inspect.signature(self.optimizer.zero_grad			).parameters
							if accept_arg:
										if set_to_none is None:
													__lowerCamelCase  							=      False
										self.optimizer.zero_grad(set_to_none=__UpperCAmelCase			)
							else:
										if set_to_none is not None:
													raise ValueError('''`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.'''			)
										self.optimizer.zero_grad()




	def     lowerCamelCase (  self		,			__UpperCAmelCase=None			):



				'''simple docstring'''
				if self.gradient_state.sync_gradients:
							if self.accelerator_state.distributed_type == DistributedType.TPU:
										__lowerCamelCase  							=      {'''closure''': closure} if closure is not None else {}
										xm.optimizer_step(self.optimizer		,			optimizer_args=__UpperCAmelCase			)
							elif self.scaler is not None:
										__lowerCamelCase  							=      False
										if self._last_scale is None:
													# `get_scale` is an async operation requiring full synchronization
													# on CPU and GPUs before finishing. As a result, we store away
													# the prior one to reduce the call overhead
													__lowerCamelCase  							=      self.scaler.get_scale()
													__lowerCamelCase  							=      True
										self.scaler.step(self.optimizer		,			__UpperCAmelCase			)
										self.scaler.update()
										__lowerCamelCase  							=      self.scaler.get_scale()
										if not new_scale:
													# If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
													__lowerCamelCase  							=      scale_after < self._last_scale
										__lowerCamelCase  							=      scale_after

							else:
										self.optimizer.step(__UpperCAmelCase			)




	def     lowerCamelCase (  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				for param_group in self.optimizer.param_groups:
							__lowerCamelCase  							=      [parameters_map.get(__UpperCAmelCase		,			__UpperCAmelCase			) for p in param_group['''params''']]




	@property
	def     lowerCamelCase (  self			):



				'''simple docstring'''
				warnings.warn(
				    '''The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use '''
				    '''`optimizer.step_was_skipped` instead.'''		,			__UpperCAmelCase		,			)
				return self._is_overflow




	@property
	def     lowerCamelCase (  self			):



				'''simple docstring'''
				return self._is_overflow




	def __getstate__(  self			):



				'''simple docstring'''
				return self.__dict__.copy()




	def __setstate__(  self		,			__UpperCAmelCase			):



				'''simple docstring'''
				self.__dict__.update(__UpperCAmelCase			)











import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
  from transformers.models.electra.modeling_flax_electra import (
      FlaxElectraForCausalLM,
      FlaxElectraForMaskedLM,
      FlaxElectraForMultipleChoice,
      FlaxElectraForPreTraining,
      FlaxElectraForQuestionAnswering,
      FlaxElectraForSequenceClassification,
      FlaxElectraForTokenClassification,
      FlaxElectraModel,
  )



class 	__lowerCAmelCase       (		unittest.TestCase				):




 def __init__(  self		,			__UpperCAmelCase		,			__UpperCAmelCase=13		,			__UpperCAmelCase=7		,			__UpperCAmelCase=True		,			__UpperCAmelCase=True		,			__UpperCAmelCase=True		,			__UpperCAmelCase=True		,			__UpperCAmelCase=99		,			__UpperCAmelCase=24		,			__UpperCAmelCase=32		,			__UpperCAmelCase=5		,			__UpperCAmelCase=4		,			__UpperCAmelCase=37		,			__UpperCAmelCase="gelu"		,			__UpperCAmelCase=0.1		,			__UpperCAmelCase=0.1		,			__UpperCAmelCase=512		,			__UpperCAmelCase=16		,			__UpperCAmelCase=2		,			__UpperCAmelCase=0.02		,			__UpperCAmelCase=4		,			):



    '''simple docstring'''
    __lowerCamelCase  							=      parent
    __lowerCamelCase  							=      batch_size
    __lowerCamelCase  							=      seq_length
    __lowerCamelCase  							=      is_training
    __lowerCamelCase  							=      use_attention_mask
    __lowerCamelCase  							=      use_token_type_ids
    __lowerCamelCase  							=      use_labels
    __lowerCamelCase  							=      vocab_size
    __lowerCamelCase  							=      hidden_size
    __lowerCamelCase  							=      embedding_size
    __lowerCamelCase  							=      num_hidden_layers
    __lowerCamelCase  							=      num_attention_heads
    __lowerCamelCase  							=      intermediate_size
    __lowerCamelCase  							=      hidden_act
    __lowerCamelCase  							=      hidden_dropout_prob
    __lowerCamelCase  							=      attention_probs_dropout_prob
    __lowerCamelCase  							=      max_position_embeddings
    __lowerCamelCase  							=      type_vocab_size
    __lowerCamelCase  							=      type_sequence_label_size
    __lowerCamelCase  							=      initializer_range
    __lowerCamelCase  							=      num_choices




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      ids_tensor([self.batch_size, self.seq_length]		,			self.vocab_size			)

    __lowerCamelCase  							=      None
    if self.use_attention_mask:
       __lowerCamelCase  							=      random_attention_mask([self.batch_size, self.seq_length]			)

    __lowerCamelCase  							=      None
    if self.use_token_type_ids:
       __lowerCamelCase  							=      ids_tensor([self.batch_size, self.seq_length]		,			self.type_vocab_size			)

    __lowerCamelCase  							=      ElectraConfig(
        vocab_size=self.vocab_size		,			hidden_size=self.hidden_size		,			embedding_size=self.embedding_size		,			num_hidden_layers=self.num_hidden_layers		,			num_attention_heads=self.num_attention_heads		,			intermediate_size=self.intermediate_size		,			hidden_act=self.hidden_act		,			hidden_dropout_prob=self.hidden_dropout_prob		,			attention_probs_dropout_prob=self.attention_probs_dropout_prob		,			max_position_embeddings=self.max_position_embeddings		,			type_vocab_size=self.type_vocab_size		,			initializer_range=self.initializer_range		,			)

    return config, input_ids, token_type_ids, attention_mask




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      self.prepare_config_and_inputs()
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase  							=      config_and_inputs
    __lowerCamelCase  							=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
    return config, inputs_dict



@require_flax
class 	__lowerCAmelCase       (		lowerCAmelCase__					,		unittest.TestCase				):
 lowerCAmelCase__       =					True

 lowerCAmelCase__       =					(
     (
         FlaxElectraModel,
         FlaxElectraForCausalLM,
         FlaxElectraForMaskedLM,
         FlaxElectraForPreTraining,
         FlaxElectraForTokenClassification,
         FlaxElectraForQuestionAnswering,
         FlaxElectraForMultipleChoice,
         FlaxElectraForSequenceClassification,
     )
     if is_flax_available()
     else ()
 )




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      FlaxElectraModelTester(self			)




 @slow
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    for model_class_name in self.all_model_classes:
       if model_class_name == FlaxElectraForMaskedLM:
          __lowerCamelCase  							=      model_class_name.from_pretrained('''google/electra-small-generator'''			)
       else:
          __lowerCamelCase  							=      model_class_name.from_pretrained('''google/electra-small-discriminator'''			)
       __lowerCamelCase  							=      model(np.ones((1, 1)			)			)
       self.assertIsNotNone(__UpperCAmelCase			)






