


'''simple docstring'''

import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a_  :    List[str]      =			16
a_  :    Union[str, Any]      =			32





def __snake_case (				UpperCAmelCase_						:						Union[str, Any]		):
 return int(x / 2**20		)





class 					snake_case   :





  """simple docstring"""




  def __enter__(   self					):







   """simple docstring"""
   gc.collect()
   torch.cuda.empty_cache()
   torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
   lowerCamelCase_								=  torch.cuda.memory_allocated()
   lowerCamelCase_								=  psutil.Process()

   lowerCamelCase_								=  self.cpu_mem_used()
   lowerCamelCase_								=  True
   lowerCamelCase_								=  threading.Thread(target=self.peak_monitor_func					)
   lowerCamelCase_								=  True
   peak_monitor_thread.start()
   return self






  def   snake_case		(   self					):







   """simple docstring"""
   return self.process.memory_info().rss






  def   snake_case		(   self					):







   """simple docstring"""
   lowerCamelCase_								=  -1

   while True:
    lowerCamelCase_								=  max(self.cpu_mem_used()      ,  self.cpu_peak					)

    # can't sleep or will not catch the peak right (this comment is here on purpose)
    # time.sleep(0.001) # 1msec

    if not self.peak_monitoring:
     break







  def __exit__(   self      ,  *UpperCamelCase					):







   """simple docstring"""
   lowerCamelCase_								=  False

   gc.collect()
   torch.cuda.empty_cache()
   lowerCamelCase_								=  torch.cuda.memory_allocated()
   lowerCamelCase_								=  torch.cuda.max_memory_allocated()
   lowerCamelCase_								=  bamb(self.end - self.begin					)
   lowerCamelCase_								=  bamb(self.peak - self.begin					)

   lowerCamelCase_								=  self.cpu_mem_used()
   lowerCamelCase_								=  bamb(self.cpu_end - self.cpu_begin					)
   lowerCamelCase_								=  bamb(self.cpu_peak - self.cpu_begin					)
   # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
     from accelerate.test_utils.training import mocked_dataloaders

     a_  :    int      =			mocked_dataloaders  # noqa: F811





def __snake_case (				UpperCAmelCase_						:						Any ,    UpperCAmelCase_						:						int		):
 # For testing only
 if os.environ.get("TESTING_MOCKED_DATALOADERS" ,    UpperCAmelCase_		) == "1":
  lowerCamelCase_								=  2

 # New Code #
 # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
 lowerCamelCase_								=  FullyShardedDataParallelPlugin(
     state_dict_config=FullStateDictConfig(offload_to_cpu=UpperCAmelCase_ ,    ranka_only=UpperCAmelCase_		) ,    optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=UpperCAmelCase_ ,    ranka_only=UpperCAmelCase_		) ,    )

 # Initialize accelerator
 if args.with_tracking:
  lowerCamelCase_								=  Accelerator(
      cpu=args.cpu ,    mixed_precision=args.mixed_precision ,    log_with="wandb" ,    project_dir=args.logging_dir ,    fsdp_plugin=UpperCAmelCase_ ,    )
 else:
  lowerCamelCase_								=  Accelerator(fsdp_plugin=UpperCAmelCase_		)
 accelerator.print(accelerator.distributed_type		)

 if hasattr(args.checkpointing_steps ,    "isdigit"		):
  if args.checkpointing_steps == "epoch":
   lowerCamelCase_								=  args.checkpointing_steps
  elif args.checkpointing_steps.isdigit():
   lowerCamelCase_								=  int(args.checkpointing_steps		)
  else:
   raise ValueError(
       F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'''		)
 else:
  lowerCamelCase_								=  None
 # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
 lowerCamelCase_								=  config["lr"]
 lowerCamelCase_								=  int(config["num_epochs"]		)
 lowerCamelCase_								=  int(config["seed"]		)
 lowerCamelCase_								=  int(config["batch_size"]		)

 # We need to initialize the trackers we use, and also store our configuration
 if args.with_tracking:
  lowerCamelCase_								=  vars(UpperCAmelCase_		)
  accelerator.init_trackers("fsdp_glue_no_trainer" ,    UpperCAmelCase_		)

 lowerCamelCase_								=  AutoTokenizer.from_pretrained(args.model_name_or_path		)
 lowerCamelCase_								=  load_dataset("glue" ,    "mrpc"		)
 lowerCamelCase_								=  evaluate.load("glue" ,    "mrpc"		)

 def tokenize_function(UpperCAmelCase_						:						List[str]		):
  # max_length=None => use the model max length (it's actually the default)
  lowerCamelCase_								=  tokenizer(examples["sentence1"] ,    examples["sentence2"] ,    truncation=UpperCAmelCase_ ,    max_length=UpperCAmelCase_		)
  return outputs

 # Apply the method we just defined to all the examples in all the splits of the dataset
 # starting with the main process first:
 with accelerator.main_process_first():
  lowerCamelCase_								=  datasets.map(
      UpperCAmelCase_ ,    batched=UpperCAmelCase_ ,    remove_columns=["idx", "sentence1", "sentence2"] ,    )

 # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
 # transformers library
 lowerCamelCase_								=  tokenized_datasets.rename_column("label" ,    "labels"		)

 # If the batch size is too big we use gradient accumulation
 lowerCamelCase_								=  1
 if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
  lowerCamelCase_								=  batch_size // MAX_GPU_BATCH_SIZE
  lowerCamelCase_								=  MAX_GPU_BATCH_SIZE

 def collate_fn(UpperCAmelCase_						:						Dict		):
  # On TPU it's best to pad everything to the same length or training will be very slow.
  lowerCamelCase_								=  128 if accelerator.distributed_type == DistributedType.TPU else None
  # When using mixed precision we want round multiples of 8/16
  if accelerator.mixed_precision == "fp8":
   lowerCamelCase_								=  16
  elif accelerator.mixed_precision != "no":
   lowerCamelCase_								=  8
  else:
   lowerCamelCase_								=  None

  return tokenizer.pad(
      UpperCAmelCase_ ,    padding="longest" ,    max_length=UpperCAmelCase_ ,    pad_to_multiple_of=UpperCAmelCase_ ,    return_tensors="pt" ,    )

 # Instantiate dataloaders.
 lowerCamelCase_								=  DataLoader(
     tokenized_datasets["train"] ,    shuffle=UpperCAmelCase_ ,    collate_fn=UpperCAmelCase_ ,    batch_size=UpperCAmelCase_		)
 lowerCamelCase_								=  DataLoader(
     tokenized_datasets["validation"] ,    shuffle=UpperCAmelCase_ ,    collate_fn=UpperCAmelCase_ ,    batch_size=UpperCAmelCase_		)

 set_seed(UpperCAmelCase_		)

 # Instantiate the model (we build the model here so that the seed also control new weights initialization)
 lowerCamelCase_								=  AutoModelForSequenceClassification.from_pretrained(
     args.model_name_or_path ,    return_dict=UpperCAmelCase_ ,    low_cpu_mem_usage=UpperCAmelCase_		)

 # New Code #
 # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
 lowerCamelCase_								=  accelerator.prepare(UpperCAmelCase_		)
 accelerator.print(UpperCAmelCase_		)

 # Instantiate optimizer
 # New Code #
 # For FSDP feature, at present it doesn't support multiple parameter groups,
 # so we need to create a single parameter group for the whole model
 lowerCamelCase_								=  torch.optim.AdamW(params=model.parameters() ,    lr=UpperCAmelCase_ ,    weight_decay=2E-4		)

 # Instantiate scheduler
 lowerCamelCase_								=  get_linear_schedule_with_warmup(
     optimizer=UpperCAmelCase_ ,    num_warmup_steps=10 ,    num_training_steps=(len(UpperCAmelCase_		) * num_epochs) // gradient_accumulation_steps ,    )

 # New Code #
 # For FSDP feature, prepare everything except the model as we have already prepared the model
 # before creating the optimizer
 # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
 # prepare method.
 lowerCamelCase_			,lowerCamelCase_			,lowerCamelCase_			,lowerCamelCase_								=  accelerator.prepare(
     UpperCAmelCase_ ,    UpperCAmelCase_ ,    UpperCAmelCase_ ,    UpperCAmelCase_		)

 lowerCamelCase_								=  0

 # Potentially load in the weights and states from a previous save
 if args.resume_from_checkpoint:
  if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
   accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}'''		)
   accelerator.load_state(args.resume_from_checkpoint		)
   lowerCamelCase_								=  os.path.basename(args.resume_from_checkpoint		)
  else:
   # Get the most recent checkpoint
   lowerCamelCase_								=  [f.name for f in os.scandir(os.getcwd()		) if f.is_dir()]
   dirs.sort(key=os.path.getctime		)
   lowerCamelCase_								=  dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
  # Extract `epoch_{i}` or `step_{i}`
  lowerCamelCase_								=  os.path.splitext(UpperCAmelCase_		)[0]

  if "epoch" in training_difference:
   num_epochs -= int(training_difference.replace("epoch_" ,    ""		)		)
   lowerCamelCase_								=  None
  else:
   lowerCamelCase_								=  int(training_difference.replace("step_" ,    ""		)		)
   num_epochs -= resume_step // len(UpperCAmelCase_		)
   # If resuming by step, we also need to know exactly how far into the DataLoader we went
   lowerCamelCase_								=  (num_epochs * len(UpperCAmelCase_		)) - resume_step

    # Now we train the model
 for epoch in range(UpperCAmelCase_		):
  # New Code #
  # context manager to track the peak memory usage during the training epoch
  with TorchTracemalloc() as tracemalloc:
   model.train()
   if args.with_tracking:
    lowerCamelCase_								=  0
   for step, batch in enumerate(UpperCAmelCase_		):
    # We need to skip steps until we reach the resumed step
    if args.resume_from_checkpoint and epoch == 0:
     if resume_step is not None and step < resume_step:
      pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
    batch.to(accelerator.device		)
    lowerCamelCase_								=  model(**UpperCAmelCase_		)
    lowerCamelCase_								=  outputs.loss
    lowerCamelCase_								=  loss / gradient_accumulation_steps
    # We keep track of the loss at each epoch
    if args.with_tracking:
     total_loss += loss.detach().float()
    accelerator.backward(UpperCAmelCase_		)
    if step % gradient_accumulation_steps == 0:
     optimizer.step()
     lr_scheduler.step()
     optimizer.zero_grad()
     # accelerator.print(lr_scheduler.get_lr())

    overall_step += 1

    if isinstance(UpperCAmelCase_ ,    UpperCAmelCase_		):
     lowerCamelCase_								=  F'''step_{overall_step}'''
     if overall_step % checkpointing_steps == 0:
      if args.output_dir is not None:
       lowerCamelCase_								=  os.path.join(args.output_dir ,    UpperCAmelCase_		)
      accelerator.save_state(UpperCAmelCase_		)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
  accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin		)		)		)
  accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used		)		)
  accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked		)		)
  accelerator.print(
      "Total Peak Memory consumed during the train (max): {}".format(
          tracemalloc.peaked + bamb(tracemalloc.begin		)		)		)
  # Logging the peak memory usage of the GPU to the tracker
  if args.with_tracking:
   accelerator.log(
       {
           "train_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin		),
       } ,    step=UpperCAmelCase_ ,    )

  # New Code #
  # context manager to track the peak memory usage during the evaluation
  with TorchTracemalloc() as tracemalloc:
   model.eval()
   for step, batch in enumerate(UpperCAmelCase_		):
    # We could avoid this line since we set the accelerator with `device_placement=True`.
    batch.to(accelerator.device		)
    with torch.no_grad():
     lowerCamelCase_								=  model(**UpperCAmelCase_		)
    lowerCamelCase_								=  outputs.logits.argmax(dim=-1		)
    lowerCamelCase_			,lowerCamelCase_								=  accelerator.gather_for_metrics((predictions, batch["labels"])		)
    metric.add_batch(
        predictions=UpperCAmelCase_ ,    references=UpperCAmelCase_ ,    )

   lowerCamelCase_								=  metric.compute()
   # Use accelerator.print to print only on the main process.
   accelerator.print(F'''epoch {epoch}:''' ,    UpperCAmelCase_		)
   if args.with_tracking:
    accelerator.log(
        {
            "accuracy": eval_metric["accuracy"],
            "f1": eval_metric["f1"],
            "train_loss": total_loss.item() / len(UpperCAmelCase_		),
        } ,    step=UpperCAmelCase_ ,    )

   if checkpointing_steps == "epoch":
    lowerCamelCase_								=  F'''epoch_{epoch}'''
    if args.output_dir is not None:
     lowerCamelCase_								=  os.path.join(args.output_dir ,    UpperCAmelCase_		)
    accelerator.save_state(UpperCAmelCase_		)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
  accelerator.print("Memory before entering the eval : {}".format(bamb(tracemalloc.begin		)		)		)
  accelerator.print("Memory consumed at the end of the eval (end-begin): {}".format(tracemalloc.used		)		)
  accelerator.print("Peak Memory consumed during the eval (max-begin): {}".format(tracemalloc.peaked		)		)
  accelerator.print(
      "Total Peak Memory consumed during the eval (max): {}".format(tracemalloc.peaked + bamb(tracemalloc.begin		)		)		)
  # Logging the peak memory usage of the GPU to the tracker
  if args.with_tracking:
   accelerator.log(
       {
           "eval_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin		),
       } ,    step=UpperCAmelCase_ ,    )

 if args.with_tracking:
  accelerator.end_training()





def __snake_case (				):
 lowerCamelCase_								=  argparse.ArgumentParser(description="Simple example of training script."		)
 parser.add_argument(
     "--mixed_precision" ,    type=UpperCAmelCase_ ,    default=UpperCAmelCase_ ,    choices=["no", "fp16", "bf16", "fp8"] ,    help="Whether to use mixed precision. Choose"
     "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
     "and an Nvidia Ampere GPU." ,    )
 parser.add_argument("--cpu" ,    action="store_true" ,    help="If passed, will train on the CPU."		)
 parser.add_argument(
     "--checkpointing_steps" ,    type=UpperCAmelCase_ ,    default=UpperCAmelCase_ ,    help="Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch." ,    )
 parser.add_argument(
     "--resume_from_checkpoint" ,    type=UpperCAmelCase_ ,    default=UpperCAmelCase_ ,    help="If the training should continue from a checkpoint folder." ,    )
 parser.add_argument(
     "--with_tracking" ,    action="store_true" ,    help="Whether to load in all available experiment trackers from the environment and use them for logging." ,    )
 parser.add_argument(
     "--output_dir" ,    type=UpperCAmelCase_ ,    default="." ,    help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory." ,    )
 parser.add_argument(
     "--logging_dir" ,    type=UpperCAmelCase_ ,    default="logs" ,    help="Location on where to store experiment tracking logs`" ,    )
 parser.add_argument(
     "--model_name_or_path" ,    type=UpperCAmelCase_ ,    help="Path to pretrained model or model identifier from huggingface.co/models." ,    required=UpperCAmelCase_ ,    )
 lowerCamelCase_								=  parser.parse_args()
 lowerCamelCase_								=  {"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
 training_function(UpperCAmelCase_ ,    UpperCAmelCase_		)


if __name__ == "__main__":
     main()







'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a_  :    Optional[Any]      =			logging.get_logger(__name__)

a_  :    Tuple      =			{
    """google/electra-small-generator""": """https://huggingface.co/google/electra-small-generator/resolve/main/config.json""",
    """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/config.json""",
    """google/electra-large-generator""": """https://huggingface.co/google/electra-large-generator/resolve/main/config.json""",
    """google/electra-small-discriminator""": (
        """https://huggingface.co/google/electra-small-discriminator/resolve/main/config.json"""
    ),
    """google/electra-base-discriminator""": (
        """https://huggingface.co/google/electra-base-discriminator/resolve/main/config.json"""
    ),
    """google/electra-large-discriminator""": (
        """https://huggingface.co/google/electra-large-discriminator/resolve/main/config.json"""
    ),
}





class 					snake_case   (   lowercase      ):





  """simple docstring"""




  _lowerCamelCase		   =  "electra"






  def __init__(   self      ,  UpperCamelCase=3_0522      ,  UpperCamelCase=128      ,  UpperCamelCase=256      ,  UpperCamelCase=12      ,  UpperCamelCase=4      ,  UpperCamelCase=1024      ,  UpperCamelCase="gelu"      ,  UpperCamelCase=0.1      ,  UpperCamelCase=0.1      ,  UpperCamelCase=512      ,  UpperCamelCase=2      ,  UpperCamelCase=0.02      ,  UpperCamelCase=1e-12      ,  UpperCamelCase="first"      ,  UpperCamelCase=True      ,  UpperCamelCase="gelu"      ,  UpperCamelCase=0.1      ,  UpperCamelCase=0      ,  UpperCamelCase="absolute"      ,  UpperCamelCase=True      ,  UpperCamelCase=None      ,  **UpperCamelCase      ,  ):







   """simple docstring"""
   super().__init__(pad_token_id=UpperCamelCase      ,  **UpperCamelCase					)

   lowerCamelCase_								=  vocab_size
   lowerCamelCase_								=  embedding_size
   lowerCamelCase_								=  hidden_size
   lowerCamelCase_								=  num_hidden_layers
   lowerCamelCase_								=  num_attention_heads
   lowerCamelCase_								=  intermediate_size
   lowerCamelCase_								=  hidden_act
   lowerCamelCase_								=  hidden_dropout_prob
   lowerCamelCase_								=  attention_probs_dropout_prob
   lowerCamelCase_								=  max_position_embeddings
   lowerCamelCase_								=  type_vocab_size
   lowerCamelCase_								=  initializer_range
   lowerCamelCase_								=  layer_norm_eps

   lowerCamelCase_								=  summary_type
   lowerCamelCase_								=  summary_use_proj
   lowerCamelCase_								=  summary_activation
   lowerCamelCase_								=  summary_last_dropout
   lowerCamelCase_								=  position_embedding_type
   lowerCamelCase_								=  use_cache
   lowerCamelCase_								=  classifier_dropout





class 					snake_case   (   lowercase      ):





  """simple docstring"""




  @property
  def   snake_case		(   self					):







   """simple docstring"""
   if self.task == "multiple-choice":
    lowerCamelCase_								=  {0: "batch", 1: "choice", 2: "sequence"}
   else:
    lowerCamelCase_								=  {0: "batch", 1: "sequence"}
   return OrderedDict(
       [
           ("input_ids", dynamic_axis),
           ("attention_mask", dynamic_axis),
           ("token_type_ids", dynamic_axis),
       ]					)



