




'''simple docstring'''






import torch

from accelerate import Accelerator, DistributedType



class  A		:




				'''simple docstring'''





				def __enter__(self      )     ->	Optional[int]:
							if self.enabled:
										__UpperCamelCase				:      Optional[Any]									=       self.model.no_sync()
										self.model_sync_obj.__enter__()

							return self
				def __exit__(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase      )     ->	int:
							if self.enabled:
										# Average all models on exit
										self._sync_and_avg_model_params()
										self.model_sync_obj.__exit__(_UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase      )
				def __init__(self			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase			,  _UpperCAmelCase = True      )     ->	Dict:
							if accelerator.distributed_type not in [
							    DistributedType.NO,
							    DistributedType.MULTI_CPU,
							    DistributedType.MULTI_GPU,
							]:
										raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"      )
							__UpperCamelCase				:      Optional[Any]									=       enabled and accelerator.distributed_type != DistributedType.NO
							__UpperCamelCase				:      List[Any]									=       0
							if self.enabled:
										__UpperCamelCase				:      int									=       accelerator
										__UpperCamelCase				:      Dict									=       model
										__UpperCamelCase				:      Any									=       local_sgd_steps
				def 	a_	(self      )     ->	Union[str, Any]:
							self.num_steps += 1
							if not self.enabled:
										return

							if self.num_steps % self.local_sgd_steps == 0:
										self._sync_and_avg_model_params()





				def 	a_	(self      )     ->	str:
							self.accelerator.wait_for_everyone()
							with self.accelerator.autocast():
										for param in self.model.parameters():
													__UpperCamelCase				:      List[Any]									=       self.accelerator.reduce(param.data			,  reduction="mean"      )









'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowerCAmelCase				     = {'''configuration_visual_bert''': ['''VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''VisualBertConfig''']}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_lowerCAmelCase				     = [
				    '''VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''VisualBertForMultipleChoice''',
				    '''VisualBertForPreTraining''',
				    '''VisualBertForQuestionAnswering''',
				    '''VisualBertForRegionToPhraseAlignment''',
				    '''VisualBertForVisualReasoning''',
				    '''VisualBertLayer''',
				    '''VisualBertModel''',
				    '''VisualBertPreTrainedModel''',
				]


if TYPE_CHECKING:
				from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_visual_bert import (
								    VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    VisualBertForMultipleChoice,
								    VisualBertForPreTraining,
								    VisualBertForQuestionAnswering,
								    VisualBertForRegionToPhraseAlignment,
								    VisualBertForVisualReasoning,
								    VisualBertLayer,
								    VisualBertModel,
								    VisualBertPreTrainedModel,
								)


else:
				import sys

				_lowerCAmelCase				     = _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



