



'''simple docstring'''





import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the checkpointing capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

_UpperCamelCase							    =						16
_UpperCamelCase							    =						32

def 	a_					(							_lowerCAmelCase	,_lowerCAmelCase = 16	)				->				Any:
  __lowerCamelCase	:	List[str]   	=			AutoTokenizer.from_pretrained('bert-base-cased'	)
  __lowerCamelCase	:	int   	=			load_dataset('glue'	,'mrpc'	)

  def tokenize_function(_lowerCAmelCase	):
    # max_length=None => use the model max length (it's actually the default)
    __lowerCamelCase	:	Optional[int]   	=			tokenizer(examples['sentence1']	,examples['sentence2']	,truncation=_lowerCAmelCase	,max_length=_lowerCAmelCase	)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    __lowerCamelCase	:	Tuple   	=			datasets.map(
        _lowerCAmelCase	,batched=_lowerCAmelCase	,remove_columns=['idx', 'sentence1', 'sentence2']	,)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  __lowerCamelCase	:	Union[str, Any]   	=			tokenized_datasets.rename_column('label'	,'labels'	)

  def collate_fn(_lowerCAmelCase	):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __lowerCamelCase	:	Optional[Any]   	=			128 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      __lowerCamelCase	:	int   	=			16
    elif accelerator.mixed_precision != "no":
      __lowerCamelCase	:	Tuple   	=			8
    else:
      __lowerCamelCase	:	Tuple   	=			None

    return tokenizer.pad(
        _lowerCAmelCase	,padding='longest'	,max_length=_lowerCAmelCase	,pad_to_multiple_of=_lowerCAmelCase	,return_tensors='pt'	,)

  # Instantiate dataloaders.
  __lowerCamelCase	:	Any   	=			DataLoader(
      tokenized_datasets['train']	,shuffle=_lowerCAmelCase	,collate_fn=_lowerCAmelCase	,batch_size=_lowerCAmelCase	)
  __lowerCamelCase	:	int   	=			DataLoader(
      tokenized_datasets['validation']	,shuffle=_lowerCAmelCase	,collate_fn=_lowerCAmelCase	,batch_size=_lowerCAmelCase	)

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
   from accelerate.test_utils.training import mocked_dataloaders

   _UpperCamelCase							    =						mocked_dataloaders  # noqa: F811

def 	a_					(							_lowerCAmelCase	,_lowerCAmelCase	)				->				Dict:
  # For testing only
  if os.environ.get('TESTING_MOCKED_DATALOADERS'	,_lowerCAmelCase	) == "1":
    __lowerCamelCase	:	str   	=			2
  # Initialize accelerator
  __lowerCamelCase	:	str   	=			Accelerator(cpu=args.cpu	,mixed_precision=args.mixed_precision	)
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  __lowerCamelCase	:	str   	=			config['lr']
  __lowerCamelCase	:	Any   	=			int(config['num_epochs']	)
  __lowerCamelCase	:	Union[str, Any]   	=			int(config['seed']	)
  __lowerCamelCase	:	Tuple   	=			int(config['batch_size']	)

  # New Code #
  # Parse out whether we are saving every epoch or after a certain number of batches
  if hasattr(args.checkpointing_steps	,'isdigit'	):
    if args.checkpointing_steps == "epoch":
      __lowerCamelCase	:	Union[str, Any]   	=			args.checkpointing_steps
    elif args.checkpointing_steps.isdigit():
      __lowerCamelCase	:	int   	=			int(args.checkpointing_steps	)
    else:
      raise ValueError(
          F'Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'	)
  else:
    __lowerCamelCase	:	Dict   	=			None

  set_seed(_lowerCAmelCase	)

  __lowerCamelCase	,__lowerCamelCase	:	Union[str, Any]   	=			get_dataloaders(_lowerCAmelCase	,_lowerCAmelCase	)
  __lowerCamelCase	:	int   	=			evaluate.load('glue'	,'mrpc'	)

  # If the batch size is too big we use gradient accumulation
  __lowerCamelCase	:	Tuple   	=			1
  if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
    __lowerCamelCase	:	Optional[int]   	=			batch_size // MAX_GPU_BATCH_SIZE
    __lowerCamelCase	:	str   	=			MAX_GPU_BATCH_SIZE

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __lowerCamelCase	:	Any   	=			AutoModelForSequenceClassification.from_pretrained('bert-base-cased'	,return_dict=_lowerCAmelCase	)

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  __lowerCamelCase	:	Any   	=			model.to(accelerator.device	)

  # Instantiate optimizer
  __lowerCamelCase	:	Any   	=			AdamW(params=model.parameters()	,lr=_lowerCAmelCase	)

  # Instantiate scheduler
  __lowerCamelCase	:	str   	=			get_linear_schedule_with_warmup(
      optimizer=_lowerCAmelCase	,num_warmup_steps=100	,num_training_steps=(len(_lowerCAmelCase	) * num_epochs) // gradient_accumulation_steps	,)

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __lowerCamelCase	,__lowerCamelCase	,__lowerCamelCase	,__lowerCamelCase	,__lowerCamelCase	:	Tuple   	=			accelerator.prepare(
      _lowerCAmelCase	,_lowerCAmelCase	,_lowerCAmelCase	,_lowerCAmelCase	,_lowerCAmelCase	)

  # New Code #
  # We need to keep track of how many total steps we have iterated over
  __lowerCamelCase	:	Dict   	=			0
  # We also need to keep track of the stating epoch so files are named properly
  __lowerCamelCase	:	Optional[Any]   	=			0

  # We need to load the checkpoint back in before training here with `load_state`
  # The total number of epochs is adjusted based on where the state is being loaded from,
  # as we assume continuation of the same training script
  if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
      accelerator.print(F'Resumed from checkpoint: {args.resume_from_checkpoint}'	)
      accelerator.load_state(args.resume_from_checkpoint	)
      __lowerCamelCase	:	List[str]   	=			os.path.basename(args.resume_from_checkpoint	)
    else:
      # Get the most recent checkpoint
      __lowerCamelCase	:	Any   	=			[f.name for f in os.scandir(os.getcwd()	) if f.is_dir()]
      dirs.sort(key=os.path.getctime	)
      __lowerCamelCase	:	List[Any]   	=			dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract `epoch_{i}` or `step_{i}`
    __lowerCamelCase	:	List[str]   	=			os.path.splitext(_lowerCAmelCase	)[0]

    if "epoch" in training_difference:
      __lowerCamelCase	:	Optional[int]   	=			int(training_difference.replace('epoch_'	,''	)	) + 1
      __lowerCamelCase	:	int   	=			None
    else:
      __lowerCamelCase	:	Any   	=			int(training_difference.replace('step_'	,''	)	)
      __lowerCamelCase	:	Dict   	=			resume_step // len(_lowerCAmelCase	)
      resume_step -= starting_epoch * len(_lowerCAmelCase	)

    # Now we train the model
  for epoch in range(_lowerCAmelCase	,_lowerCAmelCase	):
    model.train()
    # New Code #
    if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
      # We need to skip steps until we reach the resumed step
      __lowerCamelCase	:	str   	=			accelerator.skip_first_batches(_lowerCAmelCase	,_lowerCAmelCase	)
      overall_step += resume_step
    else:
      # After the first iteration though, we need to go back to the original dataloader
      __lowerCamelCase	:	List[Any]   	=			train_dataloader
    for step, batch in enumerate(_lowerCAmelCase	):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device	)
      __lowerCamelCase	:	Tuple   	=			model(**_lowerCAmelCase	)
      __lowerCamelCase	:	List[str]   	=			outputs.loss
      __lowerCamelCase	:	Any   	=			loss / gradient_accumulation_steps
      accelerator.backward(_lowerCAmelCase	)
      if step % gradient_accumulation_steps == 0:
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
      # New Code #
      overall_step += 1

      # New Code #
      # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
      # These are saved to folders named `step_{overall_step}`
      # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
      # If mixed precision was used, will also save a "scalar.bin" file
      if isinstance(_lowerCAmelCase	,_lowerCAmelCase	):
        __lowerCamelCase	:	Dict   	=			F'step_{overall_step}'
        if overall_step % checkpointing_steps == 0:
          if args.output_dir is not None:
            __lowerCamelCase	:	str   	=			os.path.join(args.output_dir	,_lowerCAmelCase	)
          accelerator.save_state(_lowerCAmelCase	)

    model.eval()
    for step, batch in enumerate(_lowerCAmelCase	):
      # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
      batch.to(accelerator.device	)
      with torch.no_grad():
        __lowerCamelCase	:	Dict   	=			model(**_lowerCAmelCase	)
      __lowerCamelCase	:	int   	=			outputs.logits.argmax(dim=-1	)
      __lowerCamelCase	,__lowerCamelCase	:	List[str]   	=			accelerator.gather_for_metrics((predictions, batch['labels'])	)
      metric.add_batch(
          predictions=_lowerCAmelCase	,references=_lowerCAmelCase	,)

    __lowerCamelCase	:	List[str]   	=			metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(F'epoch {epoch}:'	,_lowerCAmelCase	)

    # New Code #
    # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
    # These are saved to folders named `epoch_{epoch}`
    # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
    # If mixed precision was used, will also save a "scalar.bin" file
    if checkpointing_steps == "epoch":
      __lowerCamelCase	:	List[Any]   	=			F'epoch_{epoch}'
      if args.output_dir is not None:
        __lowerCamelCase	:	List[str]   	=			os.path.join(args.output_dir	,_lowerCAmelCase	)
      accelerator.save_state(_lowerCAmelCase	)

def 	a_					(							)				->				Dict:
  __lowerCamelCase	:	Optional[int]   	=			argparse.ArgumentParser(description='Simple example of training script.'	)
  parser.add_argument(
      '--mixed_precision'	,type=_lowerCAmelCase	,default=_lowerCAmelCase	,choices=['no', 'fp16', 'bf16', 'fp8']	,help='Whether to use mixed precision. Choose'
      'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
      'and an Nvidia Ampere GPU.'	,)
  parser.add_argument('--cpu'	,action='store_true'	,help='If passed, will train on the CPU.'	)
  parser.add_argument(
      '--checkpointing_steps'	,type=_lowerCAmelCase	,default=_lowerCAmelCase	,help='Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'	,)
  parser.add_argument(
      '--output_dir'	,type=_lowerCAmelCase	,default='.'	,help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'	,)
  parser.add_argument(
      '--resume_from_checkpoint'	,type=_lowerCAmelCase	,default=_lowerCAmelCase	,help='If the training should continue from a checkpoint folder.'	,)
  __lowerCamelCase	:	List[str]   	=			parser.parse_args()
  __lowerCamelCase	:	Optional[Any]   	=			{'lr': 2E-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
  training_function(_lowerCAmelCase	,_lowerCAmelCase	)


if __name__ == "__main__":
   main()











'''simple docstring'''





from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier

# Load iris file
_UpperCamelCase							    =						load_iris()
iris.keys()


print(f'''Target names: \n {iris.target_names} ''')
print(f'''\n Features: \n {iris.feature_names}''')

# Train set e Test set
_UpperCamelCase    ,    _UpperCamelCase    ,    _UpperCamelCase    ,    _UpperCamelCase							    =						train_test_split(
    iris['data'], iris['target'], random_state=4
)

# KNN

_UpperCamelCase							    =						KNeighborsClassifier(n_neighbors=1)
knn.fit(X_train, y_train)

# new array to test
_UpperCamelCase							    =						[[1, 2, 1, 4], [2, 3, 4, 5]]

_UpperCamelCase							    =						knn.predict(X_new)

print(
    f'''\nNew array: \n {X_new}\n\nTarget Names Prediction: \n'''
    f''' {iris["target_names"][prediction]}'''
)






