


import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_UpperCAmelCase     :					Any       =16
_UpperCAmelCase     :					Dict       =32



def      lowerCAmelCase	(						lowerCAmelCase_	)->	Dict:
				return int(x / 2**20	)







class 			snake_case__:







						'''simple docstring'''




						def __enter__(			self							)   ->      Union[str, Any]:
										gc.collect()
										torch.cuda.empty_cache()
										torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
										lowerCAmelCase_	: Dict      			=			torch.cuda.memory_allocated()
										lowerCAmelCase_	: Union[str, Any]      			=			psutil.Process()

										lowerCAmelCase_	: Optional[int]      			=			self.cpu_mem_used()
										lowerCAmelCase_	: Optional[Any]      			=			True
										lowerCAmelCase_	: List[Any]      			=			threading.Thread(target=self.peak_monitor_func							)
										lowerCAmelCase_	: List[str]      			=			True
										peak_monitor_thread.start()
										return self




						def   lowercase_	(			self							)   ->      Union[str, Any]:
										return self.process.memory_info().rss




						def   lowercase_	(			self							)   ->      List[str]:
										lowerCAmelCase_	: Any      			=			-1

										while True:
														lowerCAmelCase_	: Optional[int]      			=			max(self.cpu_mem_used()				,  self.cpu_peak							)

														# can't sleep or will not catch the peak right (this comment is here on purpose)
														# time.sleep(0.001) # 1msec

														if not self.peak_monitoring:
																		break




						def __exit__(			self				,  *__lowercase							)   ->      Optional[int]:
										lowerCAmelCase_	: int      			=			False

										gc.collect()
										torch.cuda.empty_cache()
										lowerCAmelCase_	: Optional[Any]      			=			torch.cuda.memory_allocated()
										lowerCAmelCase_	: Any      			=			torch.cuda.max_memory_allocated()
										lowerCAmelCase_	: Union[str, Any]      			=			bamb(self.end - self.begin							)
										lowerCAmelCase_	: Dict      			=			bamb(self.peak - self.begin							)

										lowerCAmelCase_	: Tuple      			=			self.cpu_mem_used()
										lowerCAmelCase_	: Tuple      			=			bamb(self.cpu_end - self.cpu_begin							)
										lowerCAmelCase_	: Union[str, Any]      			=			bamb(self.cpu_peak - self.cpu_begin							)
										# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
						from accelerate.test_utils.training import mocked_dataloaders

						_UpperCAmelCase     :					Tuple       =mocked_dataloaders  # noqa: F811



def      lowerCAmelCase	(						lowerCAmelCase_	,				lowerCAmelCase_	)->	Any:

				if os.environ.get('''TESTING_MOCKED_DATALOADERS'''	,				lowercase_	) == "1":
								lowerCAmelCase_	: Optional[int]      			=			2

				# New Code #
				# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
				lowerCAmelCase_	: int      			=			FullyShardedDataParallelPlugin(
				    state_dict_config=FullStateDictConfig(offload_to_cpu=lowercase_	,				ranka_only=lowercase_	)	,				optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=lowercase_	,				ranka_only=lowercase_	)	,				)

				# Initialize accelerator
				if args.with_tracking:
								lowerCAmelCase_	: Tuple      			=			Accelerator(
								    cpu=args.cpu	,				mixed_precision=args.mixed_precision	,				log_with='''wandb'''	,				project_dir=args.logging_dir	,				fsdp_plugin=lowercase_	,				)
				else:
								lowerCAmelCase_	: Optional[Any]      			=			Accelerator(fsdp_plugin=lowercase_	)
				accelerator.print(accelerator.distributed_type	)

				if hasattr(args.checkpointing_steps	,				'''isdigit'''	):
								if args.checkpointing_steps == "epoch":
												lowerCAmelCase_	: List[Any]      			=			args.checkpointing_steps
								elif args.checkpointing_steps.isdigit():
												lowerCAmelCase_	: List[str]      			=			int(args.checkpointing_steps	)
								else:
												raise ValueError(
												    f"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""	)
				else:
								lowerCAmelCase_	: int      			=			None
				# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
				lowerCAmelCase_	: str      			=			config['''lr''']
				lowerCAmelCase_	: List[Any]      			=			int(config['''num_epochs''']	)
				lowerCAmelCase_	: Optional[Any]      			=			int(config['''seed''']	)
				lowerCAmelCase_	: Any      			=			int(config['''batch_size''']	)

				# We need to initialize the trackers we use, and also store our configuration
				if args.with_tracking:
								lowerCAmelCase_	: Dict      			=			vars(lowercase_	)
								accelerator.init_trackers('''fsdp_glue_no_trainer'''	,				lowercase_	)

				lowerCAmelCase_	: Tuple      			=			AutoTokenizer.from_pretrained(args.model_name_or_path	)
				lowerCAmelCase_	: Tuple      			=			load_dataset('''glue'''	,				'''mrpc'''	)
				lowerCAmelCase_	: Any      			=			evaluate.load('''glue'''	,				'''mrpc'''	)

				def tokenize_function(lowerCAmelCase_	):
								# max_length=None => use the model max length (it's actually the default)
								lowerCAmelCase_	: Any      			=			tokenizer(examples['''sentence1''']	,				examples['''sentence2''']	,				truncation=lowercase_	,				max_length=lowercase_	)
								return outputs

				# Apply the method we just defined to all the examples in all the splits of the dataset
				# starting with the main process first:
				with accelerator.main_process_first():
								lowerCAmelCase_	: str      			=			datasets.map(
								    lowercase_	,				batched=lowercase_	,				remove_columns=['''idx''', '''sentence1''', '''sentence2''']	,				)

				# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
				# transformers library
				lowerCAmelCase_	: Any      			=			tokenized_datasets.rename_column('''label'''	,				'''labels'''	)

				# If the batch size is too big we use gradient accumulation
				lowerCAmelCase_	: Dict      			=			1
				if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
								lowerCAmelCase_	: Any      			=			batch_size // MAX_GPU_BATCH_SIZE
								lowerCAmelCase_	: Any      			=			MAX_GPU_BATCH_SIZE

				def collate_fn(lowerCAmelCase_	):
								# On TPU it's best to pad everything to the same length or training will be very slow.
								lowerCAmelCase_	: Union[str, Any]      			=			128 if accelerator.distributed_type == DistributedType.TPU else None
								# When using mixed precision we want round multiples of 8/16
								if accelerator.mixed_precision == "fp8":
												lowerCAmelCase_	: Optional[Any]      			=			16
								elif accelerator.mixed_precision != "no":
												lowerCAmelCase_	: Optional[Any]      			=			8
								else:
												lowerCAmelCase_	: int      			=			None

								return tokenizer.pad(
								    lowercase_	,				padding='''longest'''	,				max_length=lowercase_	,				pad_to_multiple_of=lowercase_	,				return_tensors='''pt'''	,				)

				# Instantiate dataloaders.
				lowerCAmelCase_	: Tuple      			=			DataLoader(
				    tokenized_datasets['''train''']	,				shuffle=lowercase_	,				collate_fn=lowercase_	,				batch_size=lowercase_	)
				lowerCAmelCase_	: str      			=			DataLoader(
				    tokenized_datasets['''validation''']	,				shuffle=lowercase_	,				collate_fn=lowercase_	,				batch_size=lowercase_	)

				set_seed(lowercase_	)

				# Instantiate the model (we build the model here so that the seed also control new weights initialization)
				lowerCAmelCase_	: int      			=			AutoModelForSequenceClassification.from_pretrained(
				    args.model_name_or_path	,				return_dict=lowercase_	,				low_cpu_mem_usage=lowercase_	)

				# New Code #
				# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
				lowerCAmelCase_	: Dict      			=			accelerator.prepare(lowercase_	)
				accelerator.print(lowercase_	)

				# Instantiate optimizer
				# New Code #
				# For FSDP feature, at present it doesn't support multiple parameter groups,
				# so we need to create a single parameter group for the whole model
				lowerCAmelCase_	: List[str]      			=			torch.optim.AdamW(params=model.parameters()	,				lr=lowercase_	,				weight_decay=2e-4	)

				# Instantiate scheduler
				lowerCAmelCase_	: Union[str, Any]      			=			get_linear_schedule_with_warmup(
				    optimizer=lowercase_	,				num_warmup_steps=10	,				num_training_steps=(len(lowercase_	) * num_epochs) // gradient_accumulation_steps	,				)

				# New Code #
				# For FSDP feature, prepare everything except the model as we have already prepared the model
				# before creating the optimizer
				# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
				# prepare method.
				lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_			,					lowerCAmelCase_	: str      			=			accelerator.prepare(
				    lowercase_	,				lowercase_	,				lowercase_	,				lowercase_	)

				lowerCAmelCase_	: int      			=			0

				# Potentially load in the weights and states from a previous save
				if args.resume_from_checkpoint:
								if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
												accelerator.print(f"""Resumed from checkpoint: {args.resume_from_checkpoint}"""	)
												accelerator.load_state(args.resume_from_checkpoint	)
												lowerCAmelCase_	: Tuple      			=			os.path.basename(args.resume_from_checkpoint	)
								else:
												# Get the most recent checkpoint
												lowerCAmelCase_	: Tuple      			=			[f.name for f in os.scandir(os.getcwd()	) if f.is_dir()]
												dirs.sort(key=os.path.getctime	)
												lowerCAmelCase_	: List[Any]      			=			dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
								# Extract `epoch_{i}` or `step_{i}`
								lowerCAmelCase_	: Union[str, Any]      			=			os.path.splitext(lowercase_	)[0]

								if "epoch" in training_difference:
												num_epochs -= int(training_difference.replace('''epoch_'''	,				''''''	)	)
												lowerCAmelCase_	: Optional[Any]      			=			None
								else:
												lowerCAmelCase_	: List[str]      			=			int(training_difference.replace('''step_'''	,				''''''	)	)
												num_epochs -= resume_step // len(lowercase_	)
												# If resuming by step, we also need to know exactly how far into the DataLoader we went
												lowerCAmelCase_	: Optional[Any]      			=			(num_epochs * len(lowercase_	)) - resume_step

												 # Now we train the model
				for epoch in range(lowercase_	):
								# New Code #
								# context manager to track the peak memory usage during the training epoch
								with TorchTracemalloc() as tracemalloc:
												model.train()
												if args.with_tracking:
																lowerCAmelCase_	: List[str]      			=			0
												for step, batch in enumerate(lowercase_	):
																# We need to skip steps until we reach the resumed step
																if args.resume_from_checkpoint and epoch == 0:
																				if resume_step is not None and step < resume_step:
																								pass
																								          # We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device	)
																lowerCAmelCase_	: Tuple      			=			model(**lowercase_	)
																lowerCAmelCase_	: Optional[int]      			=			outputs.loss
																lowerCAmelCase_	: Optional[int]      			=			loss / gradient_accumulation_steps
																# We keep track of the loss at each epoch
																if args.with_tracking:
																				total_loss += loss.detach().float()
																accelerator.backward(lowercase_	)
																if step % gradient_accumulation_steps == 0:
																				optimizer.step()
																				lr_scheduler.step()
																				optimizer.zero_grad()
																				# accelerator.print(lr_scheduler.get_lr())

																overall_step += 1

																if isinstance(lowercase_	,				lowercase_	):
																				lowerCAmelCase_	: str      			=			f"""step_{overall_step}"""
																				if overall_step % checkpointing_steps == 0:
																								if args.output_dir is not None:
																												lowerCAmelCase_	: Optional[Any]      			=			os.path.join(args.output_dir	,				lowercase_	)
																								accelerator.save_state(lowercase_	)
																								  # New Code #
																								  # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
								accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin	)	)	)
								accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used	)	)
								accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked	)	)
								accelerator.print(
								    '''Total Peak Memory consumed during the train (max): {}'''.format(
								        tracemalloc.peaked + bamb(tracemalloc.begin	)	)	)
								# Logging the peak memory usage of the GPU to the tracker
								if args.with_tracking:
												accelerator.log(
												    {
												        '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin	),
												    }	,				step=lowercase_	,				)

								# New Code #
								# context manager to track the peak memory usage during the evaluation
								with TorchTracemalloc() as tracemalloc:
												model.eval()
												for step, batch in enumerate(lowercase_	):
																# We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device	)
																with torch.no_grad():
																				lowerCAmelCase_	: List[Any]      			=			model(**lowercase_	)
																lowerCAmelCase_	: Any      			=			outputs.logits.argmax(dim=-1	)
																lowerCAmelCase_			,					lowerCAmelCase_	: Optional[int]      			=			accelerator.gather_for_metrics((predictions, batch['''labels'''])	)
																metric.add_batch(
																    predictions=lowercase_	,				references=lowercase_	,				)

												lowerCAmelCase_	: Union[str, Any]      			=			metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(f"""epoch {epoch}:"""	,				lowercase_	)
												if args.with_tracking:
																accelerator.log(
																    {
																        '''accuracy''': eval_metric['''accuracy'''],
																        '''f1''': eval_metric['''f1'''],
																        '''train_loss''': total_loss.item() / len(lowercase_	),
																    }	,				step=lowercase_	,				)

												if checkpointing_steps == "epoch":
																lowerCAmelCase_	: Optional[Any]      			=			f"""epoch_{epoch}"""
																if args.output_dir is not None:
																				lowerCAmelCase_	: int      			=			os.path.join(args.output_dir	,				lowercase_	)
																accelerator.save_state(lowercase_	)
																    # New Code #
																    # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
								accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin	)	)	)
								accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used	)	)
								accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked	)	)
								accelerator.print(
								    '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin	)	)	)
								# Logging the peak memory usage of the GPU to the tracker
								if args.with_tracking:
												accelerator.log(
												    {
												        '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin	),
												    }	,				step=lowercase_	,				)

				if args.with_tracking:
								accelerator.end_training()



def      lowerCAmelCase	(						)->	Optional[int]:
				lowerCAmelCase_	: Optional[int]      			=			argparse.ArgumentParser(description='''Simple example of training script.'''	)
				parser.add_argument(
				    '''--mixed_precision'''	,				type=lowercase_	,				default=lowercase_	,				choices=['''no''', '''fp16''', '''bf16''', '''fp8''']	,				help='''Whether to use mixed precision. Choose'''
				    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
				    '''and an Nvidia Ampere GPU.'''	,				)
				parser.add_argument('''--cpu'''	,				action='''store_true'''	,				help='''If passed, will train on the CPU.'''	)
				parser.add_argument(
				    '''--checkpointing_steps'''	,				type=lowercase_	,				default=lowercase_	,				help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''	,				)
				parser.add_argument(
				    '''--resume_from_checkpoint'''	,				type=lowercase_	,				default=lowercase_	,				help='''If the training should continue from a checkpoint folder.'''	,				)
				parser.add_argument(
				    '''--with_tracking'''	,				action='''store_true'''	,				help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''	,				)
				parser.add_argument(
				    '''--output_dir'''	,				type=lowercase_	,				default='''.'''	,				help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''	,				)
				parser.add_argument(
				    '''--logging_dir'''	,				type=lowercase_	,				default='''logs'''	,				help='''Location on where to store experiment tracking logs`'''	,				)
				parser.add_argument(
				    '''--model_name_or_path'''	,				type=lowercase_	,				help='''Path to pretrained model or model identifier from huggingface.co/models.'''	,				required=lowercase_	,				)
				lowerCAmelCase_	: Dict      			=			parser.parse_args()
				lowerCAmelCase_	: Union[str, Any]      			=			{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
				training_function(lowercase_	,				lowercase_	)


if __name__ == "__main__":
						main()



import argparse
import logging
import sys
import time

import tensorflow as tf
from datasets import load_dataset

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification


if __name__ == "__main__":
						_UpperCAmelCase     :					Optional[int]       =argparse.ArgumentParser()

						# Hyperparameters sent by the client are passed as command-line arguments to the script.
						parser.add_argument("""--epochs""", type=int, default=1)
						parser.add_argument("""--per_device_train_batch_size""", type=int, default=16)
						parser.add_argument("""--per_device_eval_batch_size""", type=int, default=8)
						parser.add_argument("""--model_name_or_path""", type=str)
						parser.add_argument("""--learning_rate""", type=str, default=5E-5)
						parser.add_argument("""--do_train""", type=bool, default=True)
						parser.add_argument("""--do_eval""", type=bool, default=True)
						parser.add_argument("""--output_dir""", type=str)

						_UpperCAmelCase       ,   _UpperCAmelCase     :					List[str]       =parser.parse_known_args()

						# overwrite batch size until we have tf_glue.py
						_UpperCAmelCase     :					Optional[Any]       =16
						_UpperCAmelCase     :					List[str]       =16

						# Set up logging
						_UpperCAmelCase     :					Optional[Any]       =logging.getLogger(__name__)

						logging.basicConfig(
						    level=logging.getLevelName("""INFO"""),
						    handlers=[logging.StreamHandler(sys.stdout)],
						    format="""%(asctime)s - %(name)s - %(levelname)s - %(message)s""",
						)

						# Load model and tokenizer
						_UpperCAmelCase     :					List[str]       =TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
						_UpperCAmelCase     :					Any       =AutoTokenizer.from_pretrained(args.model_name_or_path)

						# Load dataset
						_UpperCAmelCase       ,   _UpperCAmelCase     :					Union[str, Any]       =load_dataset("""imdb""", split=["""train""", """test"""])
						_UpperCAmelCase     :					int       =train_dataset.shuffle().select(range(5000))  # smaller the size for train dataset to 5k
						_UpperCAmelCase     :					str       =test_dataset.shuffle().select(range(500))  # smaller the size for test dataset to 500

						# Preprocess train dataset
						_UpperCAmelCase     :					str       =train_dataset.map(
						    lambda e: tokenizer(e["""text"""], truncation=True, padding="""max_length"""), batched=True
						)
						train_dataset.set_format(type="""tensorflow""", columns=["""input_ids""", """attention_mask""", """label"""])

						_UpperCAmelCase     :					List[Any]       ={
						    x: train_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
						    for x in ["""input_ids""", """attention_mask"""]
						}
						_UpperCAmelCase     :					Optional[Any]       =tf.data.Dataset.from_tensor_slices((train_features, train_dataset["""label"""])).batch(
						    args.per_device_train_batch_size
						)

						# Preprocess test dataset
						_UpperCAmelCase     :					List[Any]       =test_dataset.map(
						    lambda e: tokenizer(e["""text"""], truncation=True, padding="""max_length"""), batched=True
						)
						test_dataset.set_format(type="""tensorflow""", columns=["""input_ids""", """attention_mask""", """label"""])

						_UpperCAmelCase     :					Dict       ={
						    x: test_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
						    for x in ["""input_ids""", """attention_mask"""]
						}
						_UpperCAmelCase     :					Any       =tf.data.Dataset.from_tensor_slices((test_features, test_dataset["""label"""])).batch(
						    args.per_device_eval_batch_size
						)

						# fine optimizer and loss
						_UpperCAmelCase     :					List[Any]       =tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
						_UpperCAmelCase     :					str       =tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
						_UpperCAmelCase     :					Optional[Any]       =[tf.keras.metrics.SparseCategoricalAccuracy()]
						model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

						_UpperCAmelCase     :					Tuple       =time.time()
						_UpperCAmelCase     :					List[str]       =model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.per_device_train_batch_size)
						_UpperCAmelCase     :					Optional[int]       =time.time() - start_train_time

						logger.info("""*** Train ***""")
						logger.info(f"""train_runtime = {end_train_time}""")
						for key, value in train_results.history.items():
												logger.info(f"""  {key} = {value}""")