


import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__snake_case     =		1_6
__snake_case     =		3_2






def 						_A						(						_lowercase				)			->				Tuple:


     """simple docstring"""


     return int(x / 2**20				)



class 					__lowerCamelCase	:






       """simple docstring"""




       def __enter__(	self:				Optional[Any]			):






            '''simple docstring'''




            gc.collect()
            torch.cuda.empty_cache()
            torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
            __UpperCamelCase				=	torch.cuda.memory_allocated()
            __UpperCamelCase				=	psutil.Process()

            __UpperCamelCase				=	self.cpu_mem_used()
            __UpperCamelCase				=	True
            __UpperCamelCase				=	threading.Thread(target=self.peak_monitor_func			)
            __UpperCamelCase				=	True
            peak_monitor_thread.start()
            return self







       def 						snake_case_					(	self:				Dict			):






            '''simple docstring'''




            return self.process.memory_info().rss







       def 						snake_case_					(	self:				Union[str, Any]			):






            '''simple docstring'''




            __UpperCamelCase				=	-1

            while True:
                 __UpperCamelCase				=	max(self.cpu_mem_used(),self.cpu_peak			)

                 # can't sleep or will not catch the peak right (this comment is here on purpose)
                 # time.sleep(0.001) # 1msec

                 if not self.peak_monitoring:
                      break







       def __exit__(	self:				Optional[Any],*A_:				Dict			):






            '''simple docstring'''




            __UpperCamelCase				=	False

            gc.collect()
            torch.cuda.empty_cache()
            __UpperCamelCase				=	torch.cuda.memory_allocated()
            __UpperCamelCase				=	torch.cuda.max_memory_allocated()
            __UpperCamelCase				=	bamb(self.end - self.begin			)
            __UpperCamelCase				=	bamb(self.peak - self.begin			)

            __UpperCamelCase				=	self.cpu_mem_used()
            __UpperCamelCase				=	bamb(self.cpu_end - self.cpu_begin			)
            __UpperCamelCase				=	bamb(self.cpu_peak - self.cpu_begin			)
            # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
    from accelerate.test_utils.training import mocked_dataloaders

    __snake_case     =		mocked_dataloaders  # noqa: F811






def 						_A						(						_lowercase			,       _lowercase				)			->				Tuple:


     """simple docstring"""


     if os.environ.get('TESTING_MOCKED_DATALOADERS'			,       _A				) == "1":
          __UpperCamelCase				=	2

     # New Code #
     # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
     __UpperCamelCase				=	FullyShardedDataParallelPlugin(
         state_dict_config=FullStateDictConfig(offload_to_cpu=_A			,       ranka_only=_A				)			,       optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_A			,       ranka_only=_A				)			,       )

     # Initialize accelerator
     if args.with_tracking:
          __UpperCamelCase				=	Accelerator(
              cpu=args.cpu			,       mixed_precision=args.mixed_precision			,       log_with='wandb'			,       project_dir=args.logging_dir			,       fsdp_plugin=_A			,       )
     else:
          __UpperCamelCase				=	Accelerator(fsdp_plugin=_A				)
     accelerator.print(accelerator.distributed_type				)

     if hasattr(args.checkpointing_steps			,       'isdigit'				):
          if args.checkpointing_steps == "epoch":
               __UpperCamelCase				=	args.checkpointing_steps
          elif args.checkpointing_steps.isdigit():
               __UpperCamelCase				=	int(args.checkpointing_steps				)
          else:
               raise ValueError(
                   f'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'''				)
     else:
          __UpperCamelCase				=	None
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     __UpperCamelCase				=	config['lr']
     __UpperCamelCase				=	int(config['num_epochs']				)
     __UpperCamelCase				=	int(config['seed']				)
     __UpperCamelCase				=	int(config['batch_size']				)

     # We need to initialize the trackers we use, and also store our configuration
     if args.with_tracking:
          __UpperCamelCase				=	vars(_A				)
          accelerator.init_trackers('fsdp_glue_no_trainer'			,       _A				)

     __UpperCamelCase				=	AutoTokenizer.from_pretrained(args.model_name_or_path				)
     __UpperCamelCase				=	load_dataset('glue'			,       'mrpc'				)
     __UpperCamelCase				=	evaluate.load('glue'			,       'mrpc'				)

     def tokenize_function(_lowercase				):
          # max_length=None => use the model max length (it's actually the default)
          __UpperCamelCase				=	tokenizer(examples['sentence1']			,       examples['sentence2']			,       truncation=_A			,       max_length=_A				)
          return outputs

     # Apply the method we just defined to all the examples in all the splits of the dataset
     # starting with the main process first:
     with accelerator.main_process_first():
          __UpperCamelCase				=	datasets.map(
              _A			,       batched=_A			,       remove_columns=['idx', 'sentence1', 'sentence2']			,       )

     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
     __UpperCamelCase				=	tokenized_datasets.rename_column('label'			,       'labels'				)

     # If the batch size is too big we use gradient accumulation
     __UpperCamelCase				=	1
     if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
          __UpperCamelCase				=	batch_size // MAX_GPU_BATCH_SIZE
          __UpperCamelCase				=	MAX_GPU_BATCH_SIZE

     def collate_fn(_lowercase				):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          __UpperCamelCase				=	1_28 if accelerator.distributed_type == DistributedType.TPU else None
          # When using mixed precision we want round multiples of 8/16
          if accelerator.mixed_precision == "fp8":
               __UpperCamelCase				=	16
          elif accelerator.mixed_precision != "no":
               __UpperCamelCase				=	8
          else:
               __UpperCamelCase				=	None

          return tokenizer.pad(
              _A			,       padding='longest'			,       max_length=_A			,       pad_to_multiple_of=_A			,       return_tensors='pt'			,       )

     # Instantiate dataloaders.
     __UpperCamelCase				=	DataLoader(
         tokenized_datasets['train']			,       shuffle=_A			,       collate_fn=_A			,       batch_size=_A				)
     __UpperCamelCase				=	DataLoader(
         tokenized_datasets['validation']			,       shuffle=_A			,       collate_fn=_A			,       batch_size=_A				)

     set_seed(_A				)

     # Instantiate the model (we build the model here so that the seed also control new weights initialization)
     __UpperCamelCase				=	AutoModelForSequenceClassification.from_pretrained(
         args.model_name_or_path			,       return_dict=_A			,       low_cpu_mem_usage=_A				)

     # New Code #
     # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
     __UpperCamelCase				=	accelerator.prepare(_A				)
     accelerator.print(_A				)

     # Instantiate optimizer
     # New Code #
     # For FSDP feature, at present it doesn't support multiple parameter groups,
     # so we need to create a single parameter group for the whole model
     __UpperCamelCase				=	torch.optim.AdamW(params=model.parameters()			,       lr=_A			,       weight_decay=2e-4				)

     # Instantiate scheduler
     __UpperCamelCase				=	get_linear_schedule_with_warmup(
         optimizer=_A			,       num_warmup_steps=10			,       num_training_steps=(len(_A				) * num_epochs) // gradient_accumulation_steps			,       )

     # New Code #
     # For FSDP feature, prepare everything except the model as we have already prepared the model
     # before creating the optimizer
     # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
     # prepare method.
     __UpperCamelCase,  __UpperCamelCase,  __UpperCamelCase,  __UpperCamelCase				=	accelerator.prepare(
         _A			,       _A			,       _A			,       _A				)

     __UpperCamelCase				=	0

     # Potentially load in the weights and states from a previous save
     if args.resume_from_checkpoint:
          if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
               accelerator.print(f'''Resumed from checkpoint: {args.resume_from_checkpoint}'''				)
               accelerator.load_state(args.resume_from_checkpoint				)
               __UpperCamelCase				=	os.path.basename(args.resume_from_checkpoint				)
          else:
               # Get the most recent checkpoint
               __UpperCamelCase				=	[f.name for f in os.scandir(os.getcwd()				) if f.is_dir()]
               dirs.sort(key=os.path.getctime				)
               __UpperCamelCase				=	dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
          # Extract `epoch_{i}` or `step_{i}`
          __UpperCamelCase				=	os.path.splitext(_A				)[0]

          if "epoch" in training_difference:
               num_epochs -= int(training_difference.replace('epoch_'			,       ''				)				)
               __UpperCamelCase				=	None
          else:
               __UpperCamelCase				=	int(training_difference.replace('step_'			,       ''				)				)
               num_epochs -= resume_step // len(_A				)
               # If resuming by step, we also need to know exactly how far into the DataLoader we went
               __UpperCamelCase				=	(num_epochs * len(_A				)) - resume_step

    # Now we train the model
     for epoch in range(_A				):
          # New Code #
          # context manager to track the peak memory usage during the training epoch
          with TorchTracemalloc() as tracemalloc:
               model.train()
               if args.with_tracking:
                    __UpperCamelCase				=	0
               for step, batch in enumerate(_A				):
                    # We need to skip steps until we reach the resumed step
                    if args.resume_from_checkpoint and epoch == 0:
                         if resume_step is not None and step < resume_step:
                              pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device				)
                    __UpperCamelCase				=	model(**_A				)
                    __UpperCamelCase				=	outputs.loss
                    __UpperCamelCase				=	loss / gradient_accumulation_steps
                    # We keep track of the loss at each epoch
                    if args.with_tracking:
                         total_loss += loss.detach().float()
                    accelerator.backward(_A				)
                    if step % gradient_accumulation_steps == 0:
                         optimizer.step()
                         lr_scheduler.step()
                         optimizer.zero_grad()
                         # accelerator.print(lr_scheduler.get_lr())

                    overall_step += 1

                    if isinstance(_A			,       _A				):
                         __UpperCamelCase				=	f'''step_{overall_step}'''
                         if overall_step % checkpointing_steps == 0:
                              if args.output_dir is not None:
                                   __UpperCamelCase				=	os.path.join(args.output_dir			,       _A				)
                              accelerator.save_state(_A				)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
          accelerator.print('Memory before entering the train : {}'.format(bamb(tracemalloc.begin				)				)				)
          accelerator.print('Memory consumed at the end of the train (end-begin): {}'.format(tracemalloc.used				)				)
          accelerator.print('Peak Memory consumed during the train (max-begin): {}'.format(tracemalloc.peaked				)				)
          accelerator.print(
              'Total Peak Memory consumed during the train (max): {}'.format(
                  tracemalloc.peaked + bamb(tracemalloc.begin				)				)				)
          # Logging the peak memory usage of the GPU to the tracker
          if args.with_tracking:
               accelerator.log(
                   {
                       'train_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin				),
                   }			,       step=_A			,       )

          # New Code #
          # context manager to track the peak memory usage during the evaluation
          with TorchTracemalloc() as tracemalloc:
               model.eval()
               for step, batch in enumerate(_A				):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device				)
                    with torch.no_grad():
                         __UpperCamelCase				=	model(**_A				)
                    __UpperCamelCase				=	outputs.logits.argmax(dim=-1				)
                    __UpperCamelCase,  __UpperCamelCase				=	accelerator.gather_for_metrics((predictions, batch['labels'])				)
                    metric.add_batch(
                        predictions=_A			,       references=_A			,       )

               __UpperCamelCase				=	metric.compute()
               # Use accelerator.print to print only on the main process.
               accelerator.print(f'''epoch {epoch}:'''			,       _A				)
               if args.with_tracking:
                    accelerator.log(
                        {
                            'accuracy': eval_metric['accuracy'],
                            'f1': eval_metric['f1'],
                            'train_loss': total_loss.item() / len(_A				),
                        }			,       step=_A			,       )

               if checkpointing_steps == "epoch":
                    __UpperCamelCase				=	f'''epoch_{epoch}'''
                    if args.output_dir is not None:
                         __UpperCamelCase				=	os.path.join(args.output_dir			,       _A				)
                    accelerator.save_state(_A				)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
          accelerator.print('Memory before entering the eval : {}'.format(bamb(tracemalloc.begin				)				)				)
          accelerator.print('Memory consumed at the end of the eval (end-begin): {}'.format(tracemalloc.used				)				)
          accelerator.print('Peak Memory consumed during the eval (max-begin): {}'.format(tracemalloc.peaked				)				)
          accelerator.print(
              'Total Peak Memory consumed during the eval (max): {}'.format(tracemalloc.peaked + bamb(tracemalloc.begin				)				)				)
          # Logging the peak memory usage of the GPU to the tracker
          if args.with_tracking:
               accelerator.log(
                   {
                       'eval_total_peak_memory': tracemalloc.peaked + bamb(tracemalloc.begin				),
                   }			,       step=_A			,       )

     if args.with_tracking:
          accelerator.end_training()






def 						_A						(						)			->				Optional[Any]:


     """simple docstring"""


     __UpperCamelCase				=	argparse.ArgumentParser(description='Simple example of training script.'				)
     parser.add_argument(
         '--mixed_precision'			,       type=_A			,       default=_A			,       choices=['no', 'fp16', 'bf16', 'fp8']			,       help='Whether to use mixed precision. Choose'
         'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
         'and an Nvidia Ampere GPU.'			,       )
     parser.add_argument('--cpu'			,       action='store_true'			,       help='If passed, will train on the CPU.'				)
     parser.add_argument(
         '--checkpointing_steps'			,       type=_A			,       default=_A			,       help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'			,       )
     parser.add_argument(
         '--resume_from_checkpoint'			,       type=_A			,       default=_A			,       help='If the training should continue from a checkpoint folder.'			,       )
     parser.add_argument(
         '--with_tracking'			,       action='store_true'			,       help='Whether to load in all available experiment trackers from the environment and use them for logging.'			,       )
     parser.add_argument(
         '--output_dir'			,       type=_A			,       default='.'			,       help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'			,       )
     parser.add_argument(
         '--logging_dir'			,       type=_A			,       default='logs'			,       help='Location on where to store experiment tracking logs`'			,       )
     parser.add_argument(
         '--model_name_or_path'			,       type=_A			,       help='Path to pretrained model or model identifier from huggingface.co/models.'			,       required=_A			,       )
     __UpperCamelCase				=	parser.parse_args()
     __UpperCamelCase				=	{'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
     training_function(_A			,       _A				)


if __name__ == "__main__":
    main()






# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__snake_case     =		{
    '''configuration_convnextv2''': [
        '''CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ConvNextV2Config''',
    ]
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __snake_case     =		[
        '''CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''ConvNextV2ForImageClassification''',
        '''ConvNextV2Model''',
        '''ConvNextV2PreTrainedModel''',
        '''ConvNextV2Backbone''',
    ]


if TYPE_CHECKING:
    from .configuration_convnextva import (
        CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
        ConvNextVaConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_convnextva import (
            CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST,
            ConvNextVaBackbone,
            ConvNextVaForImageClassification,
            ConvNextVaModel,
            ConvNextVaPreTrainedModel,
        )

else:
    import sys

    __snake_case     =		_LazyModule(__name__, globals()['''__file__'''], _import_structure)


