






import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


__UpperCamelCase   :						Optional[Any]							=  get_logger(__name__)
def        a_	( _A ,				_A ,				_A ,				_A ,				_A ,				_A = None ,				)     ->     Dict:







				"""simple docstring"""







				for i, state in enumerate(_UpperCAmelCase					):
								snake_case__													=       f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								save(_UpperCAmelCase ,				_UpperCAmelCase					)
								logger.info(f'''Model weights saved in {output_model_file}'''					)
				# Optimizer states
				for i, opt in enumerate(_UpperCAmelCase					):
								snake_case__													=       opt.state_dict()
								snake_case__													=       f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								save(_UpperCAmelCase ,				_UpperCAmelCase					)
								logger.info(f'''Optimizer state saved in {output_optimizer_file}'''					)
				# Scheduler states
				for i, scheduler in enumerate(_UpperCAmelCase					):
								snake_case__													=       scheduler.state_dict()
								snake_case__													=       f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								save(_UpperCAmelCase ,				_UpperCAmelCase					)
								logger.info(f'''Scheduler state saved in {output_scheduler_file}'''					)
				# GradScaler state
				if scaler is not None:
								snake_case__													=       scaler.state_dict()
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								torch.save(_UpperCAmelCase ,				_UpperCAmelCase					)
								logger.info(f'''Gradient scaler state saved in {output_scaler_file}'''					)
				# Random number generator states
				snake_case__													=       {}
				snake_case__													=       f'''{RNG_STATE_NAME}_{process_index}.pkl'''
				snake_case__													=       random.getstate()
				snake_case__													=       np.random.get_state()
				snake_case__													=       torch.get_rng_state()
				if is_xpu_available():
								snake_case__													=       torch.xpu.get_rng_state_all()
				else:
								snake_case__													=       torch.cuda.get_rng_state_all()
				if is_tpu_available():
								snake_case__													=       xm.get_rng_state()
				snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
				torch.save(_UpperCAmelCase ,				_UpperCAmelCase					)
				logger.info(f'''Random states saved in {output_states_file}'''					)
				return output_dir
def        a_	( _A ,				_A ,				_A ,				_A ,				_A ,				_A=None ,				_A=None ,				**_A ,				)     ->     Tuple:







				"""simple docstring"""







				if map_location not in [None, "cpu", "on_device"]:
								raise TypeError(
								    'Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'					)
				if map_location is None:
								snake_case__													=       "cpu"
				elif map_location == "on_device":
								snake_case__													=       PartialState().device
				# Model states
				for i, model in enumerate(_UpperCAmelCase					):
								snake_case__													=       f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								models[i].load_state_dict(torch.load(_UpperCAmelCase ,				map_location=_UpperCAmelCase					) ,				**_UpperCAmelCase					)
				logger.info('All model weights loaded successfully'					)

				# Optimizer states
				for i, opt in enumerate(_UpperCAmelCase					):
								snake_case__													=       f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								snake_case__													=       torch.load(_UpperCAmelCase ,				map_location=_UpperCAmelCase					)
								optimizers[i].load_state_dict(_UpperCAmelCase					)
				logger.info('All optimizer states loaded successfully'					)

				# Scheduler states
				for i, scheduler in enumerate(_UpperCAmelCase					):
								snake_case__													=       f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								scheduler.load_state_dict(torch.load(_UpperCAmelCase					)					)
				logger.info('All scheduler states loaded successfully'					)

				# GradScaler state
				if scaler is not None:
								snake_case__													=       os.path.join(_UpperCAmelCase ,				_UpperCAmelCase					)
								scaler.load_state_dict(torch.load(_UpperCAmelCase					)					)
								logger.info('GradScaler state loaded successfully'					)

				# Random states
				try:
								snake_case__													=       torch.load(os.path.join(_UpperCAmelCase ,				f'''{RNG_STATE_NAME}_{process_index}.pkl'''					)					)
								random.setstate(states['random_state']					)
								np.random.set_state(states['numpy_random_seed']					)
								torch.set_rng_state(states['torch_manual_seed']					)
								if is_xpu_available():
												torch.xpu.set_rng_state_all(states['torch_xpu_manual_seed']					)
								else:
												torch.cuda.set_rng_state_all(states['torch_cuda_manual_seed']					)
								if is_tpu_available():
												xm.set_rng_state(states['xm_seed']					)
								logger.info('All random states loaded successfully'					)
				except Exception:
								logger.info('Could not load random states'					)
def        a_	( _A ,				_A ,				_A = 0					)     ->     int:







				"""simple docstring"""
				snake_case__													=       Path(_UpperCAmelCase					) / f'''custom_checkpoint_{index}.pkl'''
				logger.info(f'''Saving the state of {get_pretty_name(_UpperCAmelCase					)} to {save_location}'''					)
				torch.save(obj.state_dict() ,				_UpperCAmelCase					)


def        a_	( _A ,				_A ,				_A = 0					)     ->     Tuple:







				"""simple docstring"""
				snake_case__													=       f'''{path}/custom_checkpoint_{index}.pkl'''
				logger.info(f'''Loading the state of {get_pretty_name(_UpperCAmelCase					)} from {load_location}'''					)
				obj.load_state_dict(torch.load(_UpperCAmelCase ,				map_location='cpu'					)					)












import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
							from transformers.models.electra.modeling_flax_electra import (
							    FlaxElectraForCausalLM,
							    FlaxElectraForMaskedLM,
							    FlaxElectraForMultipleChoice,
							    FlaxElectraForPreTraining,
							    FlaxElectraForQuestionAnswering,
							    FlaxElectraForSequenceClassification,
							    FlaxElectraForTokenClassification,
							    FlaxElectraModel,
							)


class 					__SCREAMING_SNAKE_CASE(		unittest.TestCase		):
							def __init__(       self: int    ,		UpperCamelCase: List[str]    ,		UpperCamelCase: Dict=13    ,		UpperCamelCase: Dict=7    ,		UpperCamelCase: Union[str, Any]=True    ,		UpperCamelCase: List[Any]=True    ,		UpperCamelCase: Optional[int]=True    ,		UpperCamelCase: Union[str, Any]=True    ,		UpperCamelCase: Optional[int]=99    ,		UpperCamelCase: Optional[int]=24    ,		UpperCamelCase: Any=32    ,		UpperCamelCase: Any=5    ,		UpperCamelCase: Any=4    ,		UpperCamelCase: int=37    ,		UpperCamelCase: Union[str, Any]="gelu"    ,		UpperCamelCase: str=0.1    ,		UpperCamelCase: str=0.1    ,		UpperCamelCase: int=5_12    ,		UpperCamelCase: Tuple=16    ,		UpperCamelCase: Dict=2    ,		UpperCamelCase: Any=0.02    ,		UpperCamelCase: List[Any]=4    ,		)						->						int:
											snake_case__													=       parent
											snake_case__													=       batch_size
											snake_case__													=       seq_length
											snake_case__													=       is_training
											snake_case__													=       use_attention_mask
											snake_case__													=       use_token_type_ids
											snake_case__													=       use_labels
											snake_case__													=       vocab_size
											snake_case__													=       hidden_size
											snake_case__													=       embedding_size
											snake_case__													=       num_hidden_layers
											snake_case__													=       num_attention_heads
											snake_case__													=       intermediate_size
											snake_case__													=       hidden_act
											snake_case__													=       hidden_dropout_prob
											snake_case__													=       attention_probs_dropout_prob
											snake_case__													=       max_position_embeddings
											snake_case__													=       type_vocab_size
											snake_case__													=       type_sequence_label_size
											snake_case__													=       initializer_range
											snake_case__													=       num_choices
							def lowerCAmelCase_	(       self: List[str]  )						->						List[str]:
											snake_case__													=       ids_tensor([self.batch_size, self.seq_length]    ,		self.vocab_size  )

											snake_case__													=       None
											if self.use_attention_mask:
															snake_case__													=       random_attention_mask([self.batch_size, self.seq_length]  )

											snake_case__													=       None
											if self.use_token_type_ids:
															snake_case__													=       ids_tensor([self.batch_size, self.seq_length]    ,		self.type_vocab_size  )

											snake_case__													=       ElectraConfig(
											    vocab_size=self.vocab_size    ,		hidden_size=self.hidden_size    ,		embedding_size=self.embedding_size    ,		num_hidden_layers=self.num_hidden_layers    ,		num_attention_heads=self.num_attention_heads    ,		intermediate_size=self.intermediate_size    ,		hidden_act=self.hidden_act    ,		hidden_dropout_prob=self.hidden_dropout_prob    ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,		max_position_embeddings=self.max_position_embeddings    ,		type_vocab_size=self.type_vocab_size    ,		initializer_range=self.initializer_range    ,		)

											return config, input_ids, token_type_ids, attention_mask


							def lowerCAmelCase_	(       self: Tuple  )						->						Any:
											snake_case__													=       self.prepare_config_and_inputs()
											snake_case__							,  snake_case__							,  snake_case__							,  snake_case__													=       config_and_inputs
											snake_case__													=       {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
											return config, inputs_dict


@require_flax
class 					__SCREAMING_SNAKE_CASE(		a_      ,					unittest.TestCase		):
							_UpperCAmelCase	      =    True

							_UpperCAmelCase	      =    (
							    (
							        FlaxElectraModel,
							        FlaxElectraForCausalLM,
							        FlaxElectraForMaskedLM,
							        FlaxElectraForPreTraining,
							        FlaxElectraForTokenClassification,
							        FlaxElectraForQuestionAnswering,
							        FlaxElectraForMultipleChoice,
							        FlaxElectraForSequenceClassification,
							    )
							    if is_flax_available()
							    else ()
							)
							def lowerCAmelCase_	(       self: List[Any]  )						->						Dict:
											snake_case__													=       FlaxElectraModelTester(self  )


							@slow
							def lowerCAmelCase_	(       self: int  )						->						Optional[int]:
											for model_class_name in self.all_model_classes:
															if model_class_name == FlaxElectraForMaskedLM:
																			snake_case__													=       model_class_name.from_pretrained('google/electra-small-generator'  )
															else:
																			snake_case__													=       model_class_name.from_pretrained('google/electra-small-discriminator'  )
															snake_case__													=       model(np.ones((1, 1)  )  )
															self.assertIsNotNone(UpperCamelCase  )




