






import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_a				     =  16
_a				     =  32


def __A   (				__lowerCAmelCase     )->  str:





						"""simple docstring"""



						return int(x / 2**20     )





class __lowerCamelCase			:





	"""simple docstring"""






	def __enter__(							self   ):
							"""simple docstring"""







							gc.collect()
							torch.cuda.empty_cache()
							torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
							_UpperCAmelCase     =      torch.cuda.memory_allocated()
							_UpperCAmelCase     =      psutil.Process()

							_UpperCAmelCase     =      self.cpu_mem_used()
							_UpperCAmelCase     =      True
							_UpperCAmelCase     =      threading.Thread(target=self.peak_monitor_func   )
							_UpperCAmelCase     =      True
							peak_monitor_thread.start()
							return self

	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							return self.process.memory_info().rss

	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							_UpperCAmelCase     =      -1

							while True:
													_UpperCAmelCase     =      max(self.cpu_mem_used()	,   self.cpu_peak   )

													# can't sleep or will not catch the peak right (this comment is here on purpose)
													# time.sleep(0.001) # 1msec

													if not self.peak_monitoring:
																			break

	def __exit__(							self	,   *UpperCAmelCase   ):
							"""simple docstring"""







							_UpperCAmelCase     =      False

							gc.collect()
							torch.cuda.empty_cache()
							_UpperCAmelCase     =      torch.cuda.memory_allocated()
							_UpperCAmelCase     =      torch.cuda.max_memory_allocated()
							_UpperCAmelCase     =      bamb(self.end - self.begin   )
							_UpperCAmelCase     =      bamb(self.peak - self.begin   )

							_UpperCAmelCase     =      self.cpu_mem_used()
							_UpperCAmelCase     =      bamb(self.cpu_end - self.cpu_begin   )
							_UpperCAmelCase     =      bamb(self.cpu_peak - self.cpu_begin   )
							# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
		from accelerate.test_utils.training import mocked_dataloaders

		_a				     =  mocked_dataloaders  # noqa: F811


def __A   (				__lowerCAmelCase			,		__lowerCAmelCase     )->  Dict:





						"""simple docstring"""



						if os.environ.get('TESTING_MOCKED_DATALOADERS'			,		__lowerCAmelCase     ) == "1":
												_UpperCAmelCase     =      2

						# New Code #
						# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
						_UpperCAmelCase     =      FullyShardedDataParallelPlugin(
						    state_dict_config=FullStateDictConfig(offload_to_cpu=__lowerCAmelCase			,		ranka_only=__lowerCAmelCase     )			,		optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=__lowerCAmelCase			,		ranka_only=__lowerCAmelCase     )			,		)

						# Initialize accelerator
						if args.with_tracking:
												_UpperCAmelCase     =      Accelerator(
												    cpu=args.cpu			,		mixed_precision=args.mixed_precision			,		log_with='wandb'			,		project_dir=args.logging_dir			,		fsdp_plugin=__lowerCAmelCase			,		)
						else:
												_UpperCAmelCase     =      Accelerator(fsdp_plugin=__lowerCAmelCase     )
						accelerator.print(accelerator.distributed_type     )

						if hasattr(args.checkpointing_steps			,		'isdigit'     ):
												if args.checkpointing_steps == "epoch":
																		_UpperCAmelCase     =      args.checkpointing_steps
												elif args.checkpointing_steps.isdigit():
																		_UpperCAmelCase     =      int(args.checkpointing_steps     )
												else:
																		raise ValueError(
																		    F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""     )
						else:
												_UpperCAmelCase     =      None
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						_UpperCAmelCase     =      config['lr']
						_UpperCAmelCase     =      int(config['num_epochs']     )
						_UpperCAmelCase     =      int(config['seed']     )
						_UpperCAmelCase     =      int(config['batch_size']     )

						# We need to initialize the trackers we use, and also store our configuration
						if args.with_tracking:
												_UpperCAmelCase     =      vars(__lowerCAmelCase     )
												accelerator.init_trackers('fsdp_glue_no_trainer'			,		__lowerCAmelCase     )

						_UpperCAmelCase     =      AutoTokenizer.from_pretrained(args.model_name_or_path     )
						_UpperCAmelCase     =      load_dataset('glue'			,		'mrpc'     )
						_UpperCAmelCase     =      evaluate.load('glue'			,		'mrpc'     )

						def tokenize_function(__lowerCAmelCase     ):
												# max_length=None => use the model max length (it's actually the default)
												_UpperCAmelCase     =      tokenizer(examples['sentence1']			,		examples['sentence2']			,		truncation=__lowerCAmelCase			,		max_length=__lowerCAmelCase     )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												_UpperCAmelCase     =      datasets.map(
												    __lowerCAmelCase			,		batched=__lowerCAmelCase			,		remove_columns=['idx', 'sentence1', 'sentence2']			,		)

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						_UpperCAmelCase     =      tokenized_datasets.rename_column('label'			,		'labels'     )

						# If the batch size is too big we use gradient accumulation
						_UpperCAmelCase     =      1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												_UpperCAmelCase     =      batch_size // MAX_GPU_BATCH_SIZE
												_UpperCAmelCase     =      MAX_GPU_BATCH_SIZE

						def collate_fn(__lowerCAmelCase     ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												_UpperCAmelCase     =      128 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		_UpperCAmelCase     =      16
												elif accelerator.mixed_precision != "no":
																		_UpperCAmelCase     =      8
												else:
																		_UpperCAmelCase     =      None

												return tokenizer.pad(
												    __lowerCAmelCase			,		padding='longest'			,		max_length=__lowerCAmelCase			,		pad_to_multiple_of=__lowerCAmelCase			,		return_tensors='pt'			,		)

						# Instantiate dataloaders.
						_UpperCAmelCase     =      DataLoader(
						    tokenized_datasets['train']			,		shuffle=__lowerCAmelCase			,		collate_fn=__lowerCAmelCase			,		batch_size=__lowerCAmelCase     )
						_UpperCAmelCase     =      DataLoader(
						    tokenized_datasets['validation']			,		shuffle=__lowerCAmelCase			,		collate_fn=__lowerCAmelCase			,		batch_size=__lowerCAmelCase     )

						set_seed(__lowerCAmelCase     )

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						_UpperCAmelCase     =      AutoModelForSequenceClassification.from_pretrained(
						    args.model_name_or_path			,		return_dict=__lowerCAmelCase			,		low_cpu_mem_usage=__lowerCAmelCase     )

						# New Code #
						# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
						_UpperCAmelCase     =      accelerator.prepare(__lowerCAmelCase     )
						accelerator.print(__lowerCAmelCase     )

						# Instantiate optimizer
						# New Code #
						# For FSDP feature, at present it doesn't support multiple parameter groups,
						# so we need to create a single parameter group for the whole model
						_UpperCAmelCase     =      torch.optim.AdamW(params=model.parameters()			,		lr=__lowerCAmelCase			,		weight_decay=2E-4     )

						# Instantiate scheduler
						_UpperCAmelCase     =      get_linear_schedule_with_warmup(
						    optimizer=__lowerCAmelCase			,		num_warmup_steps=10			,		num_training_steps=(len(__lowerCAmelCase     ) * num_epochs) // gradient_accumulation_steps			,		)

						# New Code #
						# For FSDP feature, prepare everything except the model as we have already prepared the model
						# before creating the optimizer
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						_UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase     =      accelerator.prepare(
						    __lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase			,		__lowerCAmelCase     )

						_UpperCAmelCase     =      0

						# Potentially load in the weights and states from a previous save
						if args.resume_from_checkpoint:
												if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
																		accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""     )
																		accelerator.load_state(args.resume_from_checkpoint     )
																		_UpperCAmelCase     =      os.path.basename(args.resume_from_checkpoint     )
												else:
																		# Get the most recent checkpoint
																		_UpperCAmelCase     =      [f.name for f in os.scandir(os.getcwd()     ) if f.is_dir()]
																		dirs.sort(key=os.path.getctime     )
																		_UpperCAmelCase     =      dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
												# Extract `epoch_{i}` or `step_{i}`
												_UpperCAmelCase     =      os.path.splitext(__lowerCAmelCase     )[0]

												if "epoch" in training_difference:
																		num_epochs -= int(training_difference.replace('epoch_'			,		''     )     )
																		_UpperCAmelCase     =      None
												else:
																		_UpperCAmelCase     =      int(training_difference.replace('step_'			,		''     )     )
																		num_epochs -= resume_step // len(__lowerCAmelCase     )
																		# If resuming by step, we also need to know exactly how far into the DataLoader we went
																		_UpperCAmelCase     =      (num_epochs * len(__lowerCAmelCase     )) - resume_step

    # Now we train the model
						for epoch in range(__lowerCAmelCase     ):
												# New Code #
												# context manager to track the peak memory usage during the training epoch
												with TorchTracemalloc() as tracemalloc:
																		model.train()
																		if args.with_tracking:
																								_UpperCAmelCase     =      0
																		for step, batch in enumerate(__lowerCAmelCase     ):
																								# We need to skip steps until we reach the resumed step
																								if args.resume_from_checkpoint and epoch == 0:
																														if resume_step is not None and step < resume_step:
																																				pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device     )
																								_UpperCAmelCase     =      model(**__lowerCAmelCase     )
																								_UpperCAmelCase     =      outputs.loss
																								_UpperCAmelCase     =      loss / gradient_accumulation_steps
																								# We keep track of the loss at each epoch
																								if args.with_tracking:
																														total_loss += loss.detach().float()
																								accelerator.backward(__lowerCAmelCase     )
																								if step % gradient_accumulation_steps == 0:
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()
																														# accelerator.print(lr_scheduler.get_lr())

																								overall_step += 1

																								if isinstance(__lowerCAmelCase			,		__lowerCAmelCase     ):
																														_UpperCAmelCase     =      F"""step_{overall_step}"""
																														if overall_step % checkpointing_steps == 0:
																																				if args.output_dir is not None:
																																										_UpperCAmelCase     =      os.path.join(args.output_dir			,		__lowerCAmelCase     )
																																				accelerator.save_state(__lowerCAmelCase     )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('Memory before entering the train : {}'.format(bamb(tracemalloc.begin     )     )     )
												accelerator.print('Memory consumed at the end of the train (end-begin): {}'.format(tracemalloc.used     )     )
												accelerator.print('Peak Memory consumed during the train (max-begin): {}'.format(tracemalloc.peaked     )     )
												accelerator.print(
												    'Total Peak Memory consumed during the train (max): {}'.format(
												        tracemalloc.peaked + bamb(tracemalloc.begin     )     )     )
												# Logging the peak memory usage of the GPU to the tracker
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        'train_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin     ),
																		    }			,		step=__lowerCAmelCase			,		)

												# New Code #
												# context manager to track the peak memory usage during the evaluation
												with TorchTracemalloc() as tracemalloc:
																		model.eval()
																		for step, batch in enumerate(__lowerCAmelCase     ):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device     )
																								with torch.no_grad():
																														_UpperCAmelCase     =      model(**__lowerCAmelCase     )
																								_UpperCAmelCase     =      outputs.logits.argmax(dim=-1     )
																								_UpperCAmelCase ,			_UpperCAmelCase     =      accelerator.gather_for_metrics((predictions, batch['labels'])     )
																								metric.add_batch(
																								    predictions=__lowerCAmelCase			,		references=__lowerCAmelCase			,		)

																		_UpperCAmelCase     =      metric.compute()
																		# Use accelerator.print to print only on the main process.
																		accelerator.print(F"""epoch {epoch}:"""			,		__lowerCAmelCase     )
																		if args.with_tracking:
																								accelerator.log(
																								    {
																								        'accuracy': eval_metric['accuracy'],
																								        'f1': eval_metric['f1'],
																								        'train_loss': total_loss.item() / len(__lowerCAmelCase     ),
																								    }			,		step=__lowerCAmelCase			,		)

																		if checkpointing_steps == "epoch":
																								_UpperCAmelCase     =      F"""epoch_{epoch}"""
																								if args.output_dir is not None:
																														_UpperCAmelCase     =      os.path.join(args.output_dir			,		__lowerCAmelCase     )
																								accelerator.save_state(__lowerCAmelCase     )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('Memory before entering the eval : {}'.format(bamb(tracemalloc.begin     )     )     )
												accelerator.print('Memory consumed at the end of the eval (end-begin): {}'.format(tracemalloc.used     )     )
												accelerator.print('Peak Memory consumed during the eval (max-begin): {}'.format(tracemalloc.peaked     )     )
												accelerator.print(
												    'Total Peak Memory consumed during the eval (max): {}'.format(tracemalloc.peaked + bamb(tracemalloc.begin     )     )     )
												# Logging the peak memory usage of the GPU to the tracker
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        'eval_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin     ),
																		    }			,		step=__lowerCAmelCase			,		)

						if args.with_tracking:
												accelerator.end_training()


def __A   (				)->  Optional[Any]:





						"""simple docstring"""



						_UpperCAmelCase     =      argparse.ArgumentParser(description='Simple example of training script.'     )
						parser.add_argument(
						    '--mixed_precision'			,		type=__lowerCAmelCase			,		default=__lowerCAmelCase			,		choices=['no', 'fp16', 'bf16', 'fp8']			,		help='Whether to use mixed precision. Choose'
						    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
						    'and an Nvidia Ampere GPU.'			,		)
						parser.add_argument('--cpu'			,		action='store_true'			,		help='If passed, will train on the CPU.'     )
						parser.add_argument(
						    '--checkpointing_steps'			,		type=__lowerCAmelCase			,		default=__lowerCAmelCase			,		help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'			,		)
						parser.add_argument(
						    '--resume_from_checkpoint'			,		type=__lowerCAmelCase			,		default=__lowerCAmelCase			,		help='If the training should continue from a checkpoint folder.'			,		)
						parser.add_argument(
						    '--with_tracking'			,		action='store_true'			,		help='Whether to load in all available experiment trackers from the environment and use them for logging.'			,		)
						parser.add_argument(
						    '--output_dir'			,		type=__lowerCAmelCase			,		default='.'			,		help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'			,		)
						parser.add_argument(
						    '--logging_dir'			,		type=__lowerCAmelCase			,		default='logs'			,		help='Location on where to store experiment tracking logs`'			,		)
						parser.add_argument(
						    '--model_name_or_path'			,		type=__lowerCAmelCase			,		help='Path to pretrained model or model identifier from huggingface.co/models.'			,		required=__lowerCAmelCase			,		)
						_UpperCAmelCase     =      parser.parse_args()
						_UpperCAmelCase     =      {'lr': 2E-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
						training_function(__lowerCAmelCase			,		__lowerCAmelCase     )


if __name__ == "__main__":
		main()








import unittest

import torch

from diffusers import DDIMScheduler, TextToVideoZeroPipeline
from diffusers.utils import load_pt, require_torch_gpu, slow

from ..test_pipelines_common import assert_mean_pixel_difference





@slow
@require_torch_gpu
class __lowerCamelCase			(				unittest.TestCase):





	"""simple docstring"""






	def 	UpperCamelCase   (							self   ):
							"""simple docstring"""







							_UpperCAmelCase     =      'runwayml/stable-diffusion-v1-5'
							_UpperCAmelCase     =      TextToVideoZeroPipeline.from_pretrained(UpperCAmelCase	,   torch_dtype=torch.floataa   ).to('cuda'   )
							_UpperCAmelCase     =      DDIMScheduler.from_config(pipe.scheduler.config   )
							_UpperCAmelCase     =      torch.Generator(device='cuda'   ).manual_seed(0   )

							_UpperCAmelCase     =      'A bear is playing a guitar on Times Square'
							_UpperCAmelCase     =      pipe(prompt=UpperCAmelCase	,   generator=UpperCAmelCase   ).images

							_UpperCAmelCase     =      load_pt(
							    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text-to-video/A bear is playing a guitar on Times Square.pt'   )

							assert_mean_pixel_difference(UpperCAmelCase	,   UpperCAmelCase   )
