



import gc
import os
import tempfile
import unittest

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
from diffusers.models.attention_processor import (
    Attention,
    AttnProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRAAttnProcessor2_0,
    LoRAXFormersAttnProcessor,
    XFormersAttnProcessor,
)
from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device


def       UpperCamelCase(  __UpperCamelCase							:		Optional[Any]			):
		lowerCAmelCase_      :					Any     =			{}
		for name in unet.attn_processors.keys():
				lowerCAmelCase_      :					str     =			None if name.endswith('''attn1.processor'''			) else unet.config.cross_attention_dim
				if name.startswith('''mid_block'''			):
						lowerCAmelCase_      :					str     =			unet.config.block_out_channels[-1]
				elif name.startswith('''up_blocks'''			):
						lowerCAmelCase_      :					Tuple     =			int(name[len('''up_blocks.'''			)]			)
						lowerCAmelCase_      :					str     =			list(reversed(unet.config.block_out_channels			)			)[block_id]
				elif name.startswith('''down_blocks'''			):
						lowerCAmelCase_      :					Any     =			int(name[len('''down_blocks.'''			)]			)
						lowerCAmelCase_      :					Any     =			unet.config.block_out_channels[block_id]
				lowerCAmelCase_      :					List[Any]     =			(
				    LoRAAttnProcessor2_0 if hasattr(snake_case__					,'''scaled_dot_product_attention'''			) else LoRAAttnProcessor
				)
				lowerCAmelCase_      :					List[Any]     =			lora_attn_processor_class(
				    hidden_size=snake_case__					,cross_attention_dim=snake_case__			)
		lowerCAmelCase_      :					Optional[Any]     =			AttnProcsLayers(snake_case__			)
		return lora_attn_procs, unet_lora_layers


def       UpperCamelCase(  __UpperCamelCase							:		Any			):
		lowerCAmelCase_      :					Optional[Any]     =			{}
		lowerCAmelCase_      :					Optional[int]     =			(
		    LoRAAttnProcessor2_0 if hasattr(snake_case__					,'''scaled_dot_product_attention'''			) else LoRAAttnProcessor
		)
		for name, module in text_encoder.named_modules():
				if name.endswith(snake_case__			):
						lowerCAmelCase_      :					Any     =			lora_attn_processor_class(
						    hidden_size=module.out_proj.out_features					,cross_attention_dim=snake_case__			)
		return text_lora_attn_procs


def       UpperCamelCase(  __UpperCamelCase							:		Optional[int]			):
		lowerCAmelCase_      :					Optional[Any]     =			create_text_encoder_lora_attn_procs(snake_case__			)
		lowerCAmelCase_      :					Any     =			AttnProcsLayers(snake_case__			)
		return text_encoder_lora_layers


def       UpperCamelCase(  __UpperCamelCase							:		Any					,__UpperCamelCase							:		str=False			):
		for _, attn_proc in text_lora_attn_procs.items():
				# set up.weights
				for layer_name, layer_module in attn_proc.named_modules():
						if layer_name.endswith('''_lora'''			):
								lowerCAmelCase_      :					Any     =			(
								    torch.randn_like(layer_module.up.weight			)
								    if randn_weight
								    else torch.zeros_like(layer_module.up.weight			)
								)
								lowerCAmelCase_      :					int     =			torch.nn.Parameter(snake_case__			)





class     __snake_case						(					unittest.TestCase						):




							def        UpperCAmelCase__      (	self					:	Tuple):
									torch.manual_seed(0)
									lowerCAmelCase_      :					Any     =			UNetaDConditionModel(
									    block_out_channels=(3_2, 6_4)    ,    layers_per_block=2    ,    sample_size=3_2    ,    in_channels=4    ,    out_channels=4    ,    down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')    ,    up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')    ,    cross_attention_dim=3_2    ,    )
									lowerCAmelCase_      :					Any     =			DDIMScheduler(
									    beta_start=0.0_0085    ,    beta_end=0.012    ,    beta_schedule='''scaled_linear'''    ,    clip_sample=__SCREAMING_SNAKE_CASE    ,    set_alpha_to_one=__SCREAMING_SNAKE_CASE    ,    steps_offset=1    ,    )
									torch.manual_seed(0)
									lowerCAmelCase_      :					Dict     =			AutoencoderKL(
									    block_out_channels=[3_2, 6_4]    ,    in_channels=3    ,    out_channels=3    ,    down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']    ,    up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']    ,    latent_channels=4    ,    )
									lowerCAmelCase_      :					Dict     =			CLIPTextConfig(
									    bos_token_id=0    ,    eos_token_id=2    ,    hidden_size=3_2    ,    intermediate_size=3_7    ,    layer_norm_eps=1e-05    ,    num_attention_heads=4    ,    num_hidden_layers=5    ,    pad_token_id=1    ,    vocab_size=1_0_0_0    ,    )
									lowerCAmelCase_      :					Optional[int]     =			CLIPTextModel(__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					List[Any]     =			CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip''')

									lowerCAmelCase_ ,  lowerCAmelCase_      :					List[str]     =			create_unet_lora_layers(__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					Any     =			create_text_encoder_lora_layers(__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					Union[str, Any]     =			{
									    '''unet''': unet,
									    '''scheduler''': scheduler,
									    '''vae''': vae,
									    '''text_encoder''': text_encoder,
									    '''tokenizer''': tokenizer,
									    '''safety_checker''': None,
									    '''feature_extractor''': None,
									}
									lowerCAmelCase_      :					str     =			{
									    '''unet_lora_layers''': unet_lora_layers,
									    '''text_encoder_lora_layers''': text_encoder_lora_layers,
									    '''unet_lora_attn_procs''': unet_lora_attn_procs,
									}
									return pipeline_components, lora_components




							def        UpperCAmelCase__      (	self					:	str):
									lowerCAmelCase_      :					Optional[int]     =			1
									lowerCAmelCase_      :					Tuple     =			1_0
									lowerCAmelCase_      :					List[Any]     =			4
									lowerCAmelCase_      :					int     =			(3_2, 3_2)

									lowerCAmelCase_      :					List[Any]     =			torch.manual_seed(0)
									lowerCAmelCase_      :					List[Any]     =			floats_tensor((batch_size, num_channels) + sizes)
									lowerCAmelCase_      :					Tuple     =			torch.randint(1    ,    __SCREAMING_SNAKE_CASE    ,    size=(batch_size, sequence_length)    ,    generator=__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					List[str]     =			{
									    '''prompt''': '''A painting of a squirrel eating a burger''',
									    '''generator''': generator,
									    '''num_inference_steps''': 2,
									    '''guidance_scale''': 6.0,
									    '''output_type''': '''numpy''',
									}

									return noise, input_ids, pipeline_inputs

									# copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb




							def        UpperCAmelCase__      (	self					:	Optional[int]):
									lowerCAmelCase_      :					int     =			7_7

									lowerCAmelCase_      :					Union[str, Any]     =			torch.randint(2    ,    5_6    ,    size=(1, max_seq_length)    ,    generator=torch.manual_seed(0))

									lowerCAmelCase_      :					Optional[int]     =			{}
									lowerCAmelCase_      :					Tuple     =			inputs
									return prepared_inputs




							def        UpperCAmelCase__      (	self					:	Union[str, Any]    ,    A_					:	List[Any]):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					Optional[int]     =			self.get_dummy_components()
									LoraLoaderMixin.save_lora_weights(
									    save_directory=__SCREAMING_SNAKE_CASE    ,    unet_lora_layers=lora_components['''unet_lora_layers''']    ,    text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']    ,    )
									self.assertTrue(os.path.isfile(os.path.join(__SCREAMING_SNAKE_CASE    ,    '''pytorch_lora_weights.bin''')))




							def        UpperCAmelCase__      (	self					:	List[Any]):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					Union[str, Any]     =			self.get_dummy_components()
									lowerCAmelCase_      :					List[Any]     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					int     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
									sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_      :					Tuple     =			self.get_dummy_inputs()

									lowerCAmelCase_      :					int     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					Optional[int]     =			original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
											LoraLoaderMixin.save_lora_weights(
											    save_directory=__SCREAMING_SNAKE_CASE    ,    unet_lora_layers=lora_components['''unet_lora_layers''']    ,    text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']    ,    )
											self.assertTrue(os.path.isfile(os.path.join(__SCREAMING_SNAKE_CASE    ,    '''pytorch_lora_weights.bin''')))
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					Optional[int]     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					Optional[int]     =			lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(__SCREAMING_SNAKE_CASE)    ,    torch.from_numpy(__SCREAMING_SNAKE_CASE)))




							def        UpperCAmelCase__      (	self					:	Optional[Any]):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					str     =			self.get_dummy_components()
									lowerCAmelCase_      :					Any     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					Optional[int]     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
									sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_      :					Optional[int]     =			self.get_dummy_inputs()

									lowerCAmelCase_      :					Tuple     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					Optional[int]     =			original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
											LoraLoaderMixin.save_lora_weights(
											    save_directory=__SCREAMING_SNAKE_CASE    ,    unet_lora_layers=lora_components['''unet_lora_layers''']    ,    text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']    ,    safe_serialization=__SCREAMING_SNAKE_CASE    ,    )
											self.assertTrue(os.path.isfile(os.path.join(__SCREAMING_SNAKE_CASE    ,    '''pytorch_lora_weights.safetensors''')))
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					str     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					int     =			lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(__SCREAMING_SNAKE_CASE)    ,    torch.from_numpy(__SCREAMING_SNAKE_CASE)))




							def        UpperCAmelCase__      (	self					:	int):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					Optional[int]     =			self.get_dummy_components()
									lowerCAmelCase_      :					Union[str, Any]     =			lora_components['''unet_lora_attn_procs''']
									lowerCAmelCase_      :					Union[str, Any]     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					str     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
									sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_      :					List[Any]     =			self.get_dummy_inputs()

									lowerCAmelCase_      :					List[str]     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					List[Any]     =			original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
											lowerCAmelCase_      :					Union[str, Any]     =			sd_pipe.unet
											unet.set_attn_processor(__SCREAMING_SNAKE_CASE)
											unet.save_attn_procs(__SCREAMING_SNAKE_CASE)
											self.assertTrue(os.path.isfile(os.path.join(__SCREAMING_SNAKE_CASE    ,    '''pytorch_lora_weights.bin''')))
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					str     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					str     =			lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(__SCREAMING_SNAKE_CASE)    ,    torch.from_numpy(__SCREAMING_SNAKE_CASE)))




							def        UpperCAmelCase__      (	self					:	Union[str, Any]):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					Any     =			self.get_dummy_components()
									lowerCAmelCase_      :					str     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					str     =			self.get_dummy_tokens()

									# inference without lora
									lowerCAmelCase_      :					Any     =			pipe.text_encoder(**__SCREAMING_SNAKE_CASE)[0]
									assert outputs_without_lora.shape == (1, 7_7, 3_2)

									# create lora_attn_procs with zeroed out up.weights
									lowerCAmelCase_      :					Dict     =			create_text_encoder_lora_attn_procs(pipe.text_encoder)
									set_lora_up_weights(__SCREAMING_SNAKE_CASE    ,    randn_weight=__SCREAMING_SNAKE_CASE)

									# monkey patch
									pipe._modify_text_encoder(__SCREAMING_SNAKE_CASE)

									# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
									del text_attn_procs
									gc.collect()

									# inference with lora
									lowerCAmelCase_      :					Union[str, Any]     =			pipe.text_encoder(**__SCREAMING_SNAKE_CASE)[0]
									assert outputs_with_lora.shape == (1, 7_7, 3_2)

									assert torch.allclose(
									    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"

									# create lora_attn_procs with randn up.weights
									lowerCAmelCase_      :					Any     =			create_text_encoder_lora_attn_procs(pipe.text_encoder)
									set_lora_up_weights(__SCREAMING_SNAKE_CASE    ,    randn_weight=__SCREAMING_SNAKE_CASE)

									# monkey patch
									pipe._modify_text_encoder(__SCREAMING_SNAKE_CASE)

									# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
									del text_attn_procs
									gc.collect()

									# inference with lora
									lowerCAmelCase_      :					Optional[int]     =			pipe.text_encoder(**__SCREAMING_SNAKE_CASE)[0]
									assert outputs_with_lora.shape == (1, 7_7, 3_2)

									assert not torch.allclose(
									    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"




							def        UpperCAmelCase__      (	self					:	int):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					Optional[int]     =			self.get_dummy_components()
									lowerCAmelCase_      :					Dict     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					Optional[Any]     =			self.get_dummy_tokens()

									# inference without lora
									lowerCAmelCase_      :					Optional[Any]     =			pipe.text_encoder(**__SCREAMING_SNAKE_CASE)[0]
									assert outputs_without_lora.shape == (1, 7_7, 3_2)

									# create lora_attn_procs with randn up.weights
									lowerCAmelCase_      :					List[Any]     =			create_text_encoder_lora_attn_procs(pipe.text_encoder)
									set_lora_up_weights(__SCREAMING_SNAKE_CASE    ,    randn_weight=__SCREAMING_SNAKE_CASE)

									# monkey patch
									pipe._modify_text_encoder(__SCREAMING_SNAKE_CASE)

									# verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
									del text_attn_procs
									gc.collect()

									# inference with lora
									lowerCAmelCase_      :					int     =			pipe.text_encoder(**__SCREAMING_SNAKE_CASE)[0]
									assert outputs_with_lora.shape == (1, 7_7, 3_2)

									assert not torch.allclose(
									    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE), "lora outputs should be different to without lora outputs"

									# remove monkey patch
									pipe._remove_text_encoder_monkey_patch()

									# inference with removed lora
									lowerCAmelCase_      :					Union[str, Any]     =			pipe.text_encoder(**__SCREAMING_SNAKE_CASE)[0]
									assert outputs_without_lora_removed.shape == (1, 7_7, 3_2)

									assert torch.allclose(
									    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE), "remove lora monkey patch should restore the original outputs"




							def        UpperCAmelCase__      (	self					:	List[str]):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					Optional[Any]     =			self.get_dummy_components()
									lowerCAmelCase_      :					Any     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					str     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
									sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_      :					Union[str, Any]     =			self.get_dummy_inputs()

									with tempfile.TemporaryDirectory() as tmpdirname:
											LoraLoaderMixin.save_lora_weights(
											    save_directory=__SCREAMING_SNAKE_CASE    ,    unet_lora_layers=lora_components['''unet_lora_layers''']    ,    text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']    ,    )
											self.assertTrue(os.path.isfile(os.path.join(__SCREAMING_SNAKE_CASE    ,    '''pytorch_lora_weights.bin''')))
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					Union[str, Any]     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					Dict     =			lora_images[0, -3:, -3:, -1]

									lowerCAmelCase_      :					List[str]     =			sd_pipe(**__SCREAMING_SNAKE_CASE    ,    cross_attention_kwargs={'''scale''': 0.5}).images
									lowerCAmelCase_      :					int     =			lora_images_with_scale[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(
									    torch.allclose(torch.from_numpy(__SCREAMING_SNAKE_CASE)    ,    torch.from_numpy(__SCREAMING_SNAKE_CASE)))




							def        UpperCAmelCase__      (	self					:	Dict):
									with tempfile.TemporaryDirectory() as tmpdirname:
											self.create_lora_weight_file(__SCREAMING_SNAKE_CASE)

											lowerCAmelCase_ ,  lowerCAmelCase_      :					Optional[Any]     =			self.get_dummy_components()
											lowerCAmelCase_      :					Dict     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
											lowerCAmelCase_      :					Dict     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
											sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

											# check if vanilla attention processors are used
											for _, module in sd_pipe.unet.named_modules():
													if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE):
															self.assertIsInstance(module.processor    ,    (AttnProcessor, AttnProcessor2_0))

            # load LoRA weight file
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

											# check if lora attention processors are used
											for _, module in sd_pipe.unet.named_modules():
													if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE):
															lowerCAmelCase_      :					Tuple     =			(
															    LoRAAttnProcessor2_0 if hasattr(__SCREAMING_SNAKE_CASE    ,    '''scaled_dot_product_attention''') else LoRAAttnProcessor
															)
															self.assertIsInstance(module.processor    ,    __SCREAMING_SNAKE_CASE)




							@unittest.skipIf(torch_device != '''cuda'''    ,    '''This test is supposed to run on GPU''')
							def        UpperCAmelCase__      (	self					:	Optional[int]):
									with tempfile.TemporaryDirectory() as tmpdirname:
											self.create_lora_weight_file(__SCREAMING_SNAKE_CASE)

											lowerCAmelCase_ ,  lowerCAmelCase_      :					Union[str, Any]     =			self.get_dummy_components()
											lowerCAmelCase_      :					str     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
											lowerCAmelCase_      :					Any     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
											sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

											# enable XFormers
											sd_pipe.enable_xformers_memory_efficient_attention()

											# check if xFormers attention processors are used
											for _, module in sd_pipe.unet.named_modules():
													if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE):
															self.assertIsInstance(module.processor    ,    __SCREAMING_SNAKE_CASE)

            # load LoRA weight file
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

											# check if lora attention processors are used
											for _, module in sd_pipe.unet.named_modules():
													if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE):
															self.assertIsInstance(module.processor    ,    __SCREAMING_SNAKE_CASE)







							@unittest.skipIf(torch_device != '''cuda'''    ,    '''This test is supposed to run on GPU''')
							def        UpperCAmelCase__      (	self					:	Dict):
									lowerCAmelCase_ ,  lowerCAmelCase_      :					List[Any]     =			self.get_dummy_components()
									lowerCAmelCase_      :					List[str]     =			StableDiffusionPipeline(**__SCREAMING_SNAKE_CASE)
									lowerCAmelCase_      :					Any     =			sd_pipe.to(__SCREAMING_SNAKE_CASE)
									sd_pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_      :					Tuple     =			self.get_dummy_inputs()

									# enable XFormers
									sd_pipe.enable_xformers_memory_efficient_attention()

									lowerCAmelCase_      :					Optional[int]     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					List[Any]     =			original_images[0, -3:, -3:, -1]

									with tempfile.TemporaryDirectory() as tmpdirname:
											LoraLoaderMixin.save_lora_weights(
											    save_directory=__SCREAMING_SNAKE_CASE    ,    unet_lora_layers=lora_components['''unet_lora_layers''']    ,    text_encoder_lora_layers=lora_components['''text_encoder_lora_layers''']    ,    )
											self.assertTrue(os.path.isfile(os.path.join(__SCREAMING_SNAKE_CASE    ,    '''pytorch_lora_weights.bin''')))
											sd_pipe.load_lora_weights(__SCREAMING_SNAKE_CASE)

									lowerCAmelCase_      :					List[str]     =			sd_pipe(**__SCREAMING_SNAKE_CASE).images
									lowerCAmelCase_      :					Optional[Any]     =			lora_images[0, -3:, -3:, -1]

									# Outputs shouldn't match.
									self.assertFalse(torch.allclose(torch.from_numpy(__SCREAMING_SNAKE_CASE)    ,    torch.from_numpy(__SCREAMING_SNAKE_CASE)))










from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__  :  Tuple       = logging.get_logger(__name__)

A__  :  Tuple       = {
    '''facebook/opt-125m''': '''https://huggingface.co/facebook/opt-125m/blob/main/config.json''',
    '''facebook/opt-350m''': '''https://huggingface.co/facebook/opt-350m/blob/main/config.json''',
    '''facebook/opt-1.3b''': '''https://huggingface.co/facebook/opt-1.3b/blob/main/config.json''',
    '''facebook/opt-2.7b''': '''https://huggingface.co/facebook/opt-2.7b/blob/main/config.json''',
    '''facebook/opt-6.7b''': '''https://huggingface.co/facebook/opt-6.7b/blob/main/config.json''',
    '''facebook/opt-13b''': '''https://huggingface.co/facebook/opt-13b/blob/main/config.json''',
}



class     __snake_case						(					UpperCamelCase_						):
							_a      				=			'''opt'''
							_a      				=			['''past_key_values''']




							def __init__(	self					:	Optional[int]    ,    A_					:	Any=5_0_2_7_2    ,    A_					:	Tuple=7_6_8    ,    A_					:	Optional[Any]=1_2    ,    A_					:	str=3_0_7_2    ,    A_					:	Dict=2_0_4_8    ,    A_					:	List[Any]=True    ,    A_					:	Optional[int]=False    ,    A_					:	Optional[int]=None    ,    A_					:	Dict=0.1    ,    A_					:	str=0.0    ,    A_					:	Union[str, Any]=1_2    ,    A_					:	List[str]="relu"    ,    A_					:	Dict=0.0    ,    A_					:	Union[str, Any]=0.02    ,    A_					:	Optional[Any]=True    ,    A_					:	Optional[Any]=1    ,    A_					:	List[Any]=2    ,    A_					:	Dict=2    ,    A_					:	Optional[int]=True    ,    A_					:	Optional[int]=True    ,    **A_					:	List[Any]    ,    ):
									super().__init__(
									    pad_token_id=A_    ,    bos_token_id=A_    ,    eos_token_id=A_    ,    **A_    ,    )
									lowerCAmelCase_      :					List[Any]     =			vocab_size
									lowerCAmelCase_      :					Union[str, Any]     =			max_position_embeddings
									lowerCAmelCase_      :					Optional[int]     =			num_attention_heads
									lowerCAmelCase_      :					int     =			word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
									lowerCAmelCase_      :					List[Any]     =			ffn_dim
									lowerCAmelCase_      :					Any     =			hidden_size
									lowerCAmelCase_      :					Union[str, Any]     =			num_hidden_layers
									lowerCAmelCase_      :					str     =			dropout
									lowerCAmelCase_      :					List[str]     =			attention_dropout
									lowerCAmelCase_      :					Any     =			activation_function
									lowerCAmelCase_      :					Union[str, Any]     =			init_std
									lowerCAmelCase_      :					int     =			layerdrop
									lowerCAmelCase_      :					Optional[int]     =			use_cache
									lowerCAmelCase_      :					int     =			do_layer_norm_before
									# We keep these variables at `True` for backward compatibility.
									lowerCAmelCase_      :					Optional[Any]     =			enable_bias
									lowerCAmelCase_      :					Optional[int]     =			layer_norm_elementwise_affine

									# Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
									# with checkpoints that have been fine-tuned before transformers v4.20.1
									# see https://github.com/facebookresearch/metaseq/pull/164
									lowerCAmelCase_      :					Optional[Any]     =			_remove_final_layer_norm





