

"""simple docstring"""






import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to combine both the gradient accumulation
# and automatic batch size finder utilities of Accelerate to perfrom
# automatic gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

lowerCamelCase__			:					List[Any]							  =					32
def  UpperCamelCase   (     _lowerCAmelCase       :  Accelerator,					_lowerCAmelCase       :  int = 16	)			->     Optional[int]:
	_UpperCAmelCase      :							int       =   AutoTokenizer.from_pretrained("""bert-base-cased"""	)
	_UpperCAmelCase      :							Optional[Any]       =   load_dataset("""glue""",					"""mrpc"""	)

	def tokenize_function(_lowerCAmelCase       :  List[str]	):
		# max_length=None => use the model max length (it's actually the default)
		_UpperCAmelCase      :							Dict       =   tokenizer(examples["""sentence1"""],					examples["""sentence2"""],					truncation=_lowerCAmelCase,					max_length=_lowerCAmelCase	)
		return outputs

	# Apply the method we just defined to all the examples in all the splits of the dataset
	# starting with the main process first:
	with accelerator.main_process_first():
		_UpperCAmelCase      :							List[str]       =   datasets.map(
		    _lowerCAmelCase,					batched=_lowerCAmelCase,					remove_columns=["""idx""", """sentence1""", """sentence2"""],					)

	# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
	# transformers library
	_UpperCAmelCase      :							int       =   tokenized_datasets.rename_column("""label""",					"""labels"""	)

	def collate_fn(_lowerCAmelCase       :  Optional[int]	):
		# When using mixed precision we want round multiples of 8/16
		if accelerator.mixed_precision == "fp8":
			_UpperCAmelCase      :							str       =   16
		elif accelerator.mixed_precision != "no":
			_UpperCAmelCase      :							List[Any]       =   8
		else:
			_UpperCAmelCase      :							str       =   None

		return tokenizer.pad(
		    _lowerCAmelCase,					padding="""longest""",					pad_to_multiple_of=_lowerCAmelCase,					return_tensors="""pt""",					)

	# Instantiate dataloaders.
	_UpperCAmelCase      :							List[str]       =   DataLoader(
	    tokenized_datasets["""train"""],					shuffle=_lowerCAmelCase,					collate_fn=_lowerCAmelCase,					batch_size=_lowerCAmelCase	)
	_UpperCAmelCase      :							str       =   DataLoader(
	    tokenized_datasets["""validation"""],					shuffle=_lowerCAmelCase,					collate_fn=_lowerCAmelCase,					batch_size=_lowerCAmelCase	)

	return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
						from accelerate.test_utils.training import mocked_dataloaders

						lowerCamelCase__			:					List[str]							  =					mocked_dataloaders  # noqa: F811
def  UpperCamelCase   (     _lowerCAmelCase       :  str,					_lowerCAmelCase       :  Optional[Any]	)			->     List[Any]:
	# For testing only
	if os.environ.get("""TESTING_MOCKED_DATALOADERS""",					_lowerCAmelCase	) == "1":
		_UpperCAmelCase      :							str       =   2
	# Initialize accelerator
	_UpperCAmelCase      :							Optional[Any]       =   Accelerator(cpu=args.cpu,					mixed_precision=args.mixed_precision	)
	# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
	_UpperCAmelCase      :							Tuple       =   config["""lr"""]
	_UpperCAmelCase      :							str       =   int(config["""num_epochs"""]	)
	_UpperCAmelCase      :							Any       =   int(config["""seed"""]	)
	_UpperCAmelCase      :							Optional[int]       =   int(config["""batch_size"""]	)

	_UpperCAmelCase      :							Any       =   evaluate.load("""glue""",					"""mrpc"""	)

	# New Code #
	# We use the `find_executable_batch_size` decorator, passing in the desired observed batch size
	# to train on. If a CUDA OOM error occurs, it will retry this loop cutting the batch size in
	# half each time. From this, we can calculate the number of gradient accumulation steps needed
	# and modify the Accelerator object as a result
	@find_executable_batch_size(starting_batch_size=int(_lowerCAmelCase	)	)
	def inner_training_loop(_lowerCAmelCase       :  Any	):
		# Since we need to modify the outside accelerator object, we need to bring it
		# to the local scope
		nonlocal accelerator

		# We can calculate the number of gradient accumulation steps based on the current
		# batch size vs the starting batch size
		_UpperCAmelCase      :							str       =   observed_batch_size // batch_size

		# And then set it in the Accelerator directly:
		_UpperCAmelCase      :							Optional[Any]       =   num_gradient_accumulation_steps

		# Next we need to free all of the stored model references in the Accelerator each time
		accelerator.free_memory()

		# And set the seed so our results are reproducable each reset
		set_seed(_lowerCAmelCase	)

		# Instantiate the model (we build the model here so that the seed also control new weights initialization)
		_UpperCAmelCase      :							Optional[Any]       =   AutoModelForSequenceClassification.from_pretrained("""bert-base-cased""",					return_dict=_lowerCAmelCase	)

		# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
		# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
		# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
		_UpperCAmelCase      :							Union[str, Any]       =   model.to(accelerator.device	)

		# Instantiate optimizer
		_UpperCAmelCase      :							List[str]       =   AdamW(params=model.parameters(),					lr=_lowerCAmelCase	)
		_UpperCAmelCase  ,				_UpperCAmelCase      :							Tuple       =   get_dataloaders(_lowerCAmelCase,					_lowerCAmelCase	)

		# Instantiate scheduler
		_UpperCAmelCase      :							List[str]       =   get_linear_schedule_with_warmup(
		    optimizer=_lowerCAmelCase,					num_warmup_steps=100,					num_training_steps=(len(_lowerCAmelCase	) * num_epochs),					)

		# Prepare everything
		# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
		# prepare method.
		_UpperCAmelCase  ,				_UpperCAmelCase  ,				_UpperCAmelCase  ,				_UpperCAmelCase  ,				_UpperCAmelCase      :							List[str]       =   accelerator.prepare(
		    _lowerCAmelCase,					_lowerCAmelCase,					_lowerCAmelCase,					_lowerCAmelCase,					_lowerCAmelCase	)

		# Now we train the model
		for epoch in range(_lowerCAmelCase	):
			model.train()
			for step, batch in enumerate(_lowerCAmelCase	):
				# And perform gradient accumulation
				with accelerator.accumulate(_lowerCAmelCase	):
					# We could avoid this line since we set the accelerator with `device_placement=True`.
					batch.to(accelerator.device	)
					_UpperCAmelCase      :							Dict       =   model(**_lowerCAmelCase	)
					_UpperCAmelCase      :							str       =   outputs.loss
					accelerator.backward(_lowerCAmelCase	)
					optimizer.step()
					lr_scheduler.step()
					optimizer.zero_grad()

			model.eval()
			for step, batch in enumerate(_lowerCAmelCase	):
				# We could avoid this line since we set the accelerator with `device_placement=True`.
				batch.to(accelerator.device	)
				with torch.no_grad():
					_UpperCAmelCase      :							Tuple       =   model(**_lowerCAmelCase	)
				_UpperCAmelCase      :							Union[str, Any]       =   outputs.logits.argmax(dim=-1	)
				_UpperCAmelCase  ,				_UpperCAmelCase      :							List[str]       =   accelerator.gather_for_metrics((predictions, batch["""labels"""])	)
				metric.add_batch(
				    predictions=_lowerCAmelCase,					references=_lowerCAmelCase,					)

			_UpperCAmelCase      :							Dict       =   metric.compute()
			# Use accelerator.print to print only on the main process.
			accelerator.print(f'''epoch {epoch}:''',					_lowerCAmelCase	)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
	inner_training_loop()
def  UpperCamelCase   (     )			->     Union[str, Any]:
	_UpperCAmelCase      :							Dict       =   argparse.ArgumentParser(description="""Simple example of training script."""	)
	parser.add_argument(
	    """--mixed_precision""",					type=_lowerCAmelCase,					default=_lowerCAmelCase,					choices=["""no""", """fp16""", """bf16""", """fp8"""],					help="""Whether to use mixed precision. Choose"""
	    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
	    """and an Nvidia Ampere GPU.""",					)
	parser.add_argument("""--cpu""",					action="""store_true""",					help="""If passed, will train on the CPU."""	)
	_UpperCAmelCase      :							Dict       =   parser.parse_args()
	# New Code #
	# We modify the starting batch size to be an observed batch size of 256, to guarentee an initial CUDA OOM
	_UpperCAmelCase      :							Dict       =   {"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 256}
	training_function(_lowerCAmelCase,					_lowerCAmelCase	)


if __name__ == "__main__":
						main()








"""simple docstring"""






import unittest

import numpy as np
import torch
from torch import nn
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import KandinskyPriorPipeline, PriorTransformer, UnCLIPScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps

from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()






class       _UpperCAmelCase     (						__a      ,       unittest.TestCase):
			__a	:    List[Any]			   =							KandinskyPriorPipeline
			__a	:    List[Any]			   =							["""prompt"""]
			__a	:    List[str]			   =							["""prompt""", """negative_prompt"""]
			__a	:    Tuple			   =							[
			    """num_images_per_prompt""",
			    """generator""",
			    """num_inference_steps""",
			    """latents""",
			    """negative_prompt""",
			    """guidance_scale""",
			    """output_type""",
			    """return_dict""",
			]
			__a	:    str			   =							False
			@property
			def 							__snake_case	(						self       )      ->      Any:






				'''simple docstring'''







				return 32
			@property
			def 							__snake_case	(						self       )      ->      Union[str, Any]:






				'''simple docstring'''







				return 32
			@property
			def 							__snake_case	(						self       )      ->      Dict:






				'''simple docstring'''







				return self.time_input_dim
			@property
			def 							__snake_case	(						self       )      ->      Optional[Any]:






				'''simple docstring'''







				return self.time_input_dim * 4
			@property
			def 							__snake_case	(						self       )      ->      List[Any]:






				'''simple docstring'''







				return 1_00
			@property
			def 							__snake_case	(						self       )      ->      Optional[int]:






				'''simple docstring'''







				_UpperCAmelCase      :							List[str]       =   CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""       )
				return tokenizer
			@property
			def 							__snake_case	(						self       )      ->      int:






				'''simple docstring'''







				torch.manual_seed(0       )
				_UpperCAmelCase      :							List[Any]       =   CLIPTextConfig(
				    bos_token_id=0 ,						eos_token_id=2 ,						hidden_size=self.text_embedder_hidden_size ,						projection_dim=self.text_embedder_hidden_size ,						intermediate_size=37 ,						layer_norm_eps=1e-05 ,						num_attention_heads=4 ,						num_hidden_layers=5 ,						pad_token_id=1 ,						vocab_size=10_00 ,						)
				return CLIPTextModelWithProjection(_A       )
			@property
			def 							__snake_case	(						self       )      ->      List[Any]:






				'''simple docstring'''







				torch.manual_seed(0       )

				_UpperCAmelCase      :							Optional[int]       =   {
				    """num_attention_heads""": 2,
				    """attention_head_dim""": 12,
				    """embedding_dim""": self.text_embedder_hidden_size,
				    """num_layers""": 1,
				}

				_UpperCAmelCase      :							List[Any]       =   PriorTransformer(**_A       )
				# clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0
				_UpperCAmelCase      :							List[Any]       =   nn.Parameter(torch.ones(model.clip_std.shape       )       )
				return model
			@property
			def 							__snake_case	(						self       )      ->      Dict:






				'''simple docstring'''







				torch.manual_seed(0       )
				_UpperCAmelCase      :							Optional[Any]       =   CLIPVisionConfig(
				    hidden_size=self.text_embedder_hidden_size ,						image_size=2_24 ,						projection_dim=self.text_embedder_hidden_size ,						intermediate_size=37 ,						num_attention_heads=4 ,						num_channels=3 ,						num_hidden_layers=5 ,						patch_size=14 ,						)

				_UpperCAmelCase      :							Tuple       =   CLIPVisionModelWithProjection(_A       )
				return model
			@property
			def 							__snake_case	(						self       )      ->      List[str]:






				'''simple docstring'''







				_UpperCAmelCase      :							str       =   CLIPImageProcessor(
				    crop_size=2_24 ,						do_center_crop=_A ,						do_normalize=_A ,						do_resize=_A ,						image_mean=[0.48145466, 0.4578275, 0.40821073] ,						image_std=[0.26862954, 0.26130258, 0.27577711] ,						resample=3 ,						size=2_24 ,						)

				return image_processor
			def 							__snake_case	(						self       )      ->      Dict:






				'''simple docstring'''







				_UpperCAmelCase      :							Optional[int]       =   self.dummy_prior
				_UpperCAmelCase      :							int       =   self.dummy_image_encoder
				_UpperCAmelCase      :							int       =   self.dummy_text_encoder
				_UpperCAmelCase      :							str       =   self.dummy_tokenizer
				_UpperCAmelCase      :							Any       =   self.dummy_image_processor

				_UpperCAmelCase      :							Any       =   UnCLIPScheduler(
				    variance_type="""fixed_small_log""" ,						prediction_type="""sample""" ,						num_train_timesteps=10_00 ,						clip_sample=_A ,						clip_sample_range=10.0 ,						)

				_UpperCAmelCase      :							Any       =   {
				    """prior""": prior,
				    """image_encoder""": image_encoder,
				    """text_encoder""": text_encoder,
				    """tokenizer""": tokenizer,
				    """scheduler""": scheduler,
				    """image_processor""": image_processor,
				}

				return components
			def 							__snake_case	(						self ,						_A ,						_A=0       )      ->      Optional[Any]:






				'''simple docstring'''







				if str(_A       ).startswith("""mps"""       ):
					_UpperCAmelCase      :							Any       =   torch.manual_seed(_A       )
				else:
					_UpperCAmelCase      :							Optional[int]       =   torch.Generator(device=_A       ).manual_seed(_A       )
				_UpperCAmelCase      :							Union[str, Any]       =   {
				    """prompt""": """horse""",
				    """generator""": generator,
				    """guidance_scale""": 4.0,
				    """num_inference_steps""": 2,
				    """output_type""": """np""",
				}
				return inputs
			def 							__snake_case	(						self       )      ->      Union[str, Any]:






				'''simple docstring'''







				_UpperCAmelCase      :							List[str]       =   """cpu"""

				_UpperCAmelCase      :							int       =   self.get_dummy_components()

				_UpperCAmelCase      :							Optional[Any]       =   self.pipeline_class(**_A       )
				_UpperCAmelCase      :							List[Any]       =   pipe.to(_A       )

				pipe.set_progress_bar_config(disable=_A       )

				_UpperCAmelCase      :							str       =   pipe(**self.get_dummy_inputs(_A       )       )
				_UpperCAmelCase      :							Any       =   output.image_embeds

				_UpperCAmelCase      :							Optional[int]       =   pipe(
				    **self.get_dummy_inputs(_A       ) ,						return_dict=_A ,						)[0]

				_UpperCAmelCase      :							int       =   image[0, -10:]
				_UpperCAmelCase      :							Optional[Any]       =   image_from_tuple[0, -10:]

				assert image.shape == (1, 32)

				_UpperCAmelCase      :							Optional[Any]       =   np.array(
				    [-0.0532, 1.7120, 0.3656, -1.0852, -0.8946, -1.1756, 0.4348, 0.2482, 0.5146, -0.1156]       )

				assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1e-2
				assert np.abs(image_from_tuple_slice.flatten() - expected_slice       ).max() < 1e-2
			@skip_mps
			def 							__snake_case	(						self       )      ->      Tuple:






				'''simple docstring'''







				_UpperCAmelCase      :							str       =   torch_device == """cpu"""
				_UpperCAmelCase      :							Optional[Any]       =   True
				_UpperCAmelCase      :							List[Any]       =   False

				self._test_inference_batch_single_identical(
				    test_max_difference=_A ,						relax_max_difference=_A ,						test_mean_pixel_difference=_A ,						)
			@skip_mps
			def 							__snake_case	(						self       )      ->      Any:






				'''simple docstring'''







				_UpperCAmelCase      :							Any       =   torch_device == """cpu"""
				_UpperCAmelCase      :							List[str]       =   False

				self._test_attention_slicing_forward_pass(
				    test_max_difference=_A ,						test_mean_pixel_difference=_A ,						)





