

import torch

from accelerate import Accelerator, DistributedType







class 						__magic_name__      :

   '''simple docstring'''




   def __enter__(    self	)							->   int:






     """simple docstring"""






     if self.enabled:
       a__  =self.model.no_sync()
       self.model_sync_obj.__enter__()

     return self



   def __exit__(    self,   lowercase_,   lowercase_,   lowercase_	)							->   int:






     """simple docstring"""






     if self.enabled:
       # Average all models on exit
       self._sync_and_avg_model_params()
       self.model_sync_obj.__exit__(lowercase_,   lowercase_,   lowercase_	)



   def __init__(    self,   lowercase_,   lowercase_,   lowercase_,   lowercase_ = True	)							->   Any:






     """simple docstring"""






     if accelerator.distributed_type not in [
         DistributedType.NO,
         DistributedType.MULTI_CPU,
         DistributedType.MULTI_GPU,
     ]:
       raise NotImplementedError('''LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'''	)
     a__  =enabled and accelerator.distributed_type != DistributedType.NO
     a__  =0
     if self.enabled:
       a__  =accelerator
       a__  =model
       a__  =local_sgd_steps



   def 			_UpperCAmelCase			(    self	)							->   List[Any]:






     """simple docstring"""






     self.num_steps += 1
     if not self.enabled:
       return

     if self.num_steps % self.local_sgd_steps == 0:
       self._sync_and_avg_model_params()



   def 			_UpperCAmelCase			(    self	)							->   str:






     """simple docstring"""






     self.accelerator.wait_for_everyone()
     with self.accelerator.autocast():
       for param in self.model.parameters():
         a__  =self.accelerator.reduce(param.data,   reduction='''mean'''	)



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCamelCase									= {
    '''configuration_flaubert''': ['''FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FlaubertConfig''', '''FlaubertOnnxConfig'''],
    '''tokenization_flaubert''': ['''FlaubertTokenizer'''],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase									= [
           '''FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''FlaubertForMultipleChoice''',
           '''FlaubertForQuestionAnswering''',
           '''FlaubertForQuestionAnsweringSimple''',
           '''FlaubertForSequenceClassification''',
           '''FlaubertForTokenClassification''',
           '''FlaubertModel''',
           '''FlaubertWithLMHeadModel''',
           '''FlaubertPreTrainedModel''',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase									= [
           '''TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''TFFlaubertForMultipleChoice''',
           '''TFFlaubertForQuestionAnsweringSimple''',
           '''TFFlaubertForSequenceClassification''',
           '''TFFlaubertForTokenClassification''',
           '''TFFlaubertModel''',
           '''TFFlaubertPreTrainedModel''',
           '''TFFlaubertWithLMHeadModel''',
       ]


if TYPE_CHECKING:
       from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig, FlaubertOnnxConfig
       from .tokenization_flaubert import FlaubertTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flaubert import (
                  FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  FlaubertForMultipleChoice,
                  FlaubertForQuestionAnswering,
                  FlaubertForQuestionAnsweringSimple,
                  FlaubertForSequenceClassification,
                  FlaubertForTokenClassification,
                  FlaubertModel,
                  FlaubertPreTrainedModel,
                  FlaubertWithLMHeadModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_flaubert import (
                  TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFFlaubertForMultipleChoice,
                  TFFlaubertForQuestionAnsweringSimple,
                  TFFlaubertForSequenceClassification,
                  TFFlaubertForTokenClassification,
                  TFFlaubertModel,
                  TFFlaubertPreTrainedModel,
                  TFFlaubertWithLMHeadModel,
              )

else:
       import sys

       lowerCamelCase									= _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
