"""simple docstring"""




from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin




@dataclass
class        lowercase_  (  UpperCamelCase_						):



	"""simple docstring"""





	UpperCAmelCase_  : torch.FloatTensor




class        lowercase_  (  UpperCamelCase_				,					UpperCamelCase_						):



	"""simple docstring"""





	@register_to_config
	def __init__(							self		,      __SCREAMING_SNAKE_CASE = 16		,      __SCREAMING_SNAKE_CASE = 88		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = 1		,      __SCREAMING_SNAKE_CASE = 0.0		,      __SCREAMING_SNAKE_CASE = 32		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = False		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = "geglu"		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = False		,      __SCREAMING_SNAKE_CASE = False		,      __SCREAMING_SNAKE_CASE = False		,      __SCREAMING_SNAKE_CASE = "layer_norm"		,      __SCREAMING_SNAKE_CASE = True		,      )						->Optional[int]:
								super().__init__()
								lowerCAmelCase     		=		use_linear_projection
								lowerCAmelCase     		=		num_attention_heads
								lowerCAmelCase     		=		attention_head_dim
								lowerCAmelCase     		=		num_attention_heads * attention_head_dim

								# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
								# Define whether input is continuous or discrete depending on configuration
								lowerCAmelCase     		=		(in_channels is not None) and (patch_size is None)
								lowerCAmelCase     		=		num_vector_embeds is not None
								lowerCAmelCase     		=		in_channels is not None and patch_size is not None

								if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
															lowerCAmelCase     		=		(
															    F"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"
															    ''' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'''
															    ''' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'''
															    ''' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'''
															    ''' would be very nice if you could open a Pull request for the `transformer/config.json` file'''
															)
															deprecate('''norm_type!=num_embeds_ada_norm'''		,      '''1.0.0'''		,      __SCREAMING_SNAKE_CASE		,      standard_warn=__SCREAMING_SNAKE_CASE    )
															lowerCAmelCase     		=		'''ada_norm'''

								if self.is_input_continuous and self.is_input_vectorized:
															raise ValueError(
															    F"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"
															    ''' sure that either `in_channels` or `num_vector_embeds` is None.'''    )
								elif self.is_input_vectorized and self.is_input_patches:
															raise ValueError(
															    F"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"
															    ''' sure that either `num_vector_embeds` or `num_patches` is None.'''    )
								elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
															raise ValueError(
															    F"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"
															    F" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."    )

								# 2. Define input layers
								if self.is_input_continuous:
															lowerCAmelCase     		=		in_channels

															lowerCAmelCase     		=		torch.nn.GroupNorm(num_groups=__SCREAMING_SNAKE_CASE		,      num_channels=__SCREAMING_SNAKE_CASE		,      eps=1e-6		,      affine=__SCREAMING_SNAKE_CASE    )
															if use_linear_projection:
																						lowerCAmelCase     		=		nn.Linear(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE    )
															else:
																						lowerCAmelCase     		=		nn.Convad(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      kernel_size=1		,      stride=1		,      padding=0    )
								elif self.is_input_vectorized:
															assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
															assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

															lowerCAmelCase     		=		sample_size
															lowerCAmelCase     		=		sample_size
															lowerCAmelCase     		=		num_vector_embeds
															lowerCAmelCase     		=		self.height * self.width

															lowerCAmelCase     		=		ImagePositionalEmbeddings(
															    num_embed=__SCREAMING_SNAKE_CASE		,      embed_dim=__SCREAMING_SNAKE_CASE		,      height=self.height		,      width=self.width    )
								elif self.is_input_patches:
															assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

															lowerCAmelCase     		=		sample_size
															lowerCAmelCase     		=		sample_size

															lowerCAmelCase     		=		patch_size
															lowerCAmelCase     		=		PatchEmbed(
															    height=__SCREAMING_SNAKE_CASE		,      width=__SCREAMING_SNAKE_CASE		,      patch_size=__SCREAMING_SNAKE_CASE		,      in_channels=__SCREAMING_SNAKE_CASE		,      embed_dim=__SCREAMING_SNAKE_CASE		,      )

								# 3. Define transformers blocks
								lowerCAmelCase     		=		nn.ModuleList(
								    [
								        BasicTransformerBlock(
								            __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      dropout=__SCREAMING_SNAKE_CASE		,      cross_attention_dim=__SCREAMING_SNAKE_CASE		,      activation_fn=__SCREAMING_SNAKE_CASE		,      num_embeds_ada_norm=__SCREAMING_SNAKE_CASE		,      attention_bias=__SCREAMING_SNAKE_CASE		,      only_cross_attention=__SCREAMING_SNAKE_CASE		,      upcast_attention=__SCREAMING_SNAKE_CASE		,      norm_type=__SCREAMING_SNAKE_CASE		,      norm_elementwise_affine=__SCREAMING_SNAKE_CASE		,      )
								        for d in range(__SCREAMING_SNAKE_CASE    )
								    ]    )

								# 4. Define output layers
								lowerCAmelCase     		=		in_channels if out_channels is None else out_channels
								if self.is_input_continuous:
															# TODO: should use out_channels for continuous projections
															if use_linear_projection:
																						lowerCAmelCase     		=		nn.Linear(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE    )
															else:
																						lowerCAmelCase     		=		nn.Convad(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      kernel_size=1		,      stride=1		,      padding=0    )
								elif self.is_input_vectorized:
															lowerCAmelCase     		=		nn.LayerNorm(__SCREAMING_SNAKE_CASE    )
															lowerCAmelCase     		=		nn.Linear(__SCREAMING_SNAKE_CASE		,      self.num_vector_embeds - 1    )
								elif self.is_input_patches:
															lowerCAmelCase     		=		nn.LayerNorm(__SCREAMING_SNAKE_CASE		,      elementwise_affine=__SCREAMING_SNAKE_CASE		,      eps=1e-6    )
															lowerCAmelCase     		=		nn.Linear(__SCREAMING_SNAKE_CASE		,      2 * inner_dim    )
															lowerCAmelCase     		=		nn.Linear(__SCREAMING_SNAKE_CASE		,      patch_size * patch_size * self.out_channels    )




	def       SCREAMING_SNAKE_CASE_    (							self		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = None		,      __SCREAMING_SNAKE_CASE = True		,      )						->List[Any]:
								# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
								#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
								#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
								# expects mask of shape:
								#   [batch, key_tokens]
								# adds singleton query_tokens dimension:
								#   [batch,                    1, key_tokens]
								# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
								#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
								#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
								if attention_mask is not None and attention_mask.ndim == 2:
															# assume that mask is expressed as:
															#   (1 = keep,      0 = discard)
															# convert mask into a bias that can be added to attention scores:
															#       (keep = +0,     discard = -10000.0)
															lowerCAmelCase     		=		(1 - attention_mask.to(hidden_states.dtype    )) * -10000.0
															lowerCAmelCase     		=		attention_mask.unsqueeze(1    )

								# convert encoder_attention_mask to a bias the same way we do for attention_mask
								if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
															lowerCAmelCase     		=		(1 - encoder_attention_mask.to(hidden_states.dtype    )) * -10000.0
															lowerCAmelCase     		=		encoder_attention_mask.unsqueeze(1    )

								# 1. Input
								if self.is_input_continuous:
															lowerCAmelCase		,				lowerCAmelCase		,				lowerCAmelCase		,				lowerCAmelCase     		=		hidden_states.shape
															lowerCAmelCase     		=		hidden_states

															lowerCAmelCase     		=		self.norm(__SCREAMING_SNAKE_CASE    )
															if not self.use_linear_projection:
																						lowerCAmelCase     		=		self.proj_in(__SCREAMING_SNAKE_CASE    )
																						lowerCAmelCase     		=		hidden_states.shape[1]
																						lowerCAmelCase     		=		hidden_states.permute(0		,      2		,      3		,      1    ).reshape(__SCREAMING_SNAKE_CASE		,      height * width		,      __SCREAMING_SNAKE_CASE    )
															else:
																						lowerCAmelCase     		=		hidden_states.shape[1]
																						lowerCAmelCase     		=		hidden_states.permute(0		,      2		,      3		,      1    ).reshape(__SCREAMING_SNAKE_CASE		,      height * width		,      __SCREAMING_SNAKE_CASE    )
																						lowerCAmelCase     		=		self.proj_in(__SCREAMING_SNAKE_CASE    )
								elif self.is_input_vectorized:
															lowerCAmelCase     		=		self.latent_image_embedding(__SCREAMING_SNAKE_CASE    )
								elif self.is_input_patches:
															lowerCAmelCase     		=		self.pos_embed(__SCREAMING_SNAKE_CASE    )

								# 2. Blocks
								for block in self.transformer_blocks:
															lowerCAmelCase     		=		block(
															    __SCREAMING_SNAKE_CASE		,      attention_mask=__SCREAMING_SNAKE_CASE		,      encoder_hidden_states=__SCREAMING_SNAKE_CASE		,      encoder_attention_mask=__SCREAMING_SNAKE_CASE		,      timestep=__SCREAMING_SNAKE_CASE		,      cross_attention_kwargs=__SCREAMING_SNAKE_CASE		,      class_labels=__SCREAMING_SNAKE_CASE		,      )

								# 3. Output
								if self.is_input_continuous:
															if not self.use_linear_projection:
																						lowerCAmelCase     		=		hidden_states.reshape(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE    ).permute(0		,      3		,      1		,      2    ).contiguous()
																						lowerCAmelCase     		=		self.proj_out(__SCREAMING_SNAKE_CASE    )
															else:
																						lowerCAmelCase     		=		self.proj_out(__SCREAMING_SNAKE_CASE    )
																						lowerCAmelCase     		=		hidden_states.reshape(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE    ).permute(0		,      3		,      1		,      2    ).contiguous()

															lowerCAmelCase     		=		hidden_states + residual
								elif self.is_input_vectorized:
															lowerCAmelCase     		=		self.norm_out(__SCREAMING_SNAKE_CASE    )
															lowerCAmelCase     		=		self.out(__SCREAMING_SNAKE_CASE    )
															# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
															lowerCAmelCase     		=		logits.permute(0		,      2		,      1    )

															# log(p(x_0))
															lowerCAmelCase     		=		F.log_softmax(logits.double()		,      dim=1    ).float()
								elif self.is_input_patches:
															# TODO: cleanup!
															lowerCAmelCase     		=		self.transformer_blocks[0].norma.emb(
															    __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      hidden_dtype=hidden_states.dtype    )
															lowerCAmelCase		,				lowerCAmelCase     		=		self.proj_out_a(F.silu(__SCREAMING_SNAKE_CASE    )    ).chunk(2		,      dim=1    )
															lowerCAmelCase     		=		self.norm_out(__SCREAMING_SNAKE_CASE    ) * (1 + scale[:, None]) + shift[:, None]
															lowerCAmelCase     		=		self.proj_out_a(__SCREAMING_SNAKE_CASE    )

															# unpatchify
															lowerCAmelCase     		=		lowerCAmelCase     		=		int(hidden_states.shape[1] ** 0.5    )
															lowerCAmelCase     		=		hidden_states.reshape(
															    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)    )
															lowerCAmelCase     		=		torch.einsum('''nhwpqc->nchpwq'''		,      __SCREAMING_SNAKE_CASE    )
															lowerCAmelCase     		=		hidden_states.reshape(
															    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)    )

								if not return_dict:
															return (output,)

								return TransformeraDModelOutput(sample=__SCREAMING_SNAKE_CASE    )




import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


lowercase__      :    str          =     [nn.LayerNorm]

lowercase__      :    str          =     logging.get_logger(__name__)

lowercase__      :    List[Any]          =     version.parse(version.parse(torch.__version__).base_version)

lowercase__      :    Dict          =     parsed_torch_version_base >= version.parse('''2.0''')
lowercase__      :    int          =     parsed_torch_version_base >= version.parse('''1.12''')
lowercase__      :    Optional[int]          =     parsed_torch_version_base >= version.parse('''1.11''')
lowercase__      :    int          =     parsed_torch_version_base < version.parse('''1.11''')

def        SCREAMING_SNAKE_CASE_					(						snake_case__       ,  snake_case__       ,  snake_case__       ,  snake_case__       ,  self							)					->     Union[str, Any]:
							from torch import _softmax_backward_data

							if is_torch_less_than_1_11:
														return _softmax_backward_data(snake_case__       ,  snake_case__       ,  parent.dim       ,  self							)
							else:
														return _softmax_backward_data(snake_case__       ,  snake_case__       ,  parent.dim       ,  self.dtype							)

def        SCREAMING_SNAKE_CASE_					(						snake_case__       ,  snake_case__       ,  snake_case__ = 0							)					->     nn.Linear:
							lowerCAmelCase     		=		index.to(layer.weight.device							)
							lowerCAmelCase     		=		layer.weight.index_select(snake_case__       ,  snake_case__							).clone().detach()
							if layer.bias is not None:
														if dim == 1:
																					lowerCAmelCase     		=		layer.bias.clone().detach()
														else:
																					lowerCAmelCase     		=		layer.bias[index].clone().detach()
							lowerCAmelCase     		=		list(layer.weight.size()							)
							lowerCAmelCase     		=		len(snake_case__							)
							lowerCAmelCase     		=		nn.Linear(new_size[1]       ,  new_size[0]       ,  bias=layer.bias is not None							).to(layer.weight.device							)
							lowerCAmelCase     		=		False
							new_layer.weight.copy_(W.contiguous()							)
							lowerCAmelCase     		=		True
							if layer.bias is not None:
														lowerCAmelCase     		=		False
														new_layer.bias.copy_(b.contiguous()							)
														lowerCAmelCase     		=		True
							return new_layer




class        lowercase_  (  nn.Module						):



	"""simple docstring"""





	def __init__(							self		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE    )						->Optional[Any]:
								super().__init__()
								lowerCAmelCase     		=		nf
								lowerCAmelCase     		=		nn.Parameter(torch.empty(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE    )    )
								lowerCAmelCase     		=		nn.Parameter(torch.zeros(__SCREAMING_SNAKE_CASE    )    )
								nn.init.normal_(self.weight		,      std=0.0_2    )




	def       SCREAMING_SNAKE_CASE_    (							self		,      __SCREAMING_SNAKE_CASE    )						->List[str]:
								lowerCAmelCase     		=		x.size()[:-1] + (self.nf,)
								lowerCAmelCase     		=		torch.addmm(self.bias		,      x.view(-1		,      x.size(-1    )    )		,      self.weight    )
								lowerCAmelCase     		=		x.view(__SCREAMING_SNAKE_CASE    )
								return x

def        SCREAMING_SNAKE_CASE_					(						snake_case__       ,  snake_case__       ,  snake_case__ = 1							)					->     ConvaD:
							lowerCAmelCase     		=		index.to(layer.weight.device							)
							lowerCAmelCase     		=		layer.weight.index_select(snake_case__       ,  snake_case__							).clone().detach()
							if dim == 0:
														lowerCAmelCase     		=		layer.bias.clone().detach()
							else:
														lowerCAmelCase     		=		layer.bias[index].clone().detach()
							lowerCAmelCase     		=		list(layer.weight.size()							)
							lowerCAmelCase     		=		len(snake_case__							)
							lowerCAmelCase     		=		ConvaD(new_size[1]       ,  new_size[0]							).to(layer.weight.device							)
							lowerCAmelCase     		=		False
							new_layer.weight.copy_(W.contiguous()							)
							lowerCAmelCase     		=		True
							lowerCAmelCase     		=		False
							new_layer.bias.copy_(b.contiguous()							)
							lowerCAmelCase     		=		True
							return new_layer

def        SCREAMING_SNAKE_CASE_					(						snake_case__       ,  snake_case__       ,  snake_case__ = None							)					->     Union[nn.Linear, ConvaD]:
							if isinstance(snake_case__       ,  nn.Linear							):
														return prune_linear_layer(snake_case__       ,  snake_case__       ,  dim=0 if dim is None else dim							)
							elif isinstance(snake_case__       ,  snake_case__							):
														return prune_convad_layer(snake_case__       ,  snake_case__       ,  dim=1 if dim is None else dim							)
							else:
														raise ValueError(f"Can't prune layer of class {layer.__class__}"							)

def        SCREAMING_SNAKE_CASE_					(						snake_case__       ,  snake_case__       ,  snake_case__       ,  *snake_case__							)					->     torch.Tensor:
							assert len(snake_case__							) > 0, f"{input_tensors} has to be a tuple/list of tensors"

							# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
							lowerCAmelCase     		=		len(inspect.signature(snake_case__							).parameters							)
							if num_args_in_forward_chunk_fn != len(snake_case__							):
														raise ValueError(
														    f"forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(snake_case__							)} input "
														    '''tensors are given'''							)

							if chunk_size > 0:
														lowerCAmelCase     		=		input_tensors[0].shape[chunk_dim]
														for input_tensor in input_tensors:
																					if input_tensor.shape[chunk_dim] != tensor_shape:
																												raise ValueError(
																												    f"All input tenors have to be of the same shape: {tensor_shape}, "
																												    f"found shape {input_tensor.shape[chunk_dim]}"							)

														if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
																					raise ValueError(
																					    f"The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk "
																					    f"size {chunk_size}"							)

														lowerCAmelCase     		=		input_tensors[0].shape[chunk_dim] // chunk_size

														# chunk input tensor into tuples
														lowerCAmelCase     		=		tuple(input_tensor.chunk(snake_case__       ,  dim=snake_case__							) for input_tensor in input_tensors							)
														# apply forward fn to every tuple
														lowerCAmelCase     		=		tuple(forward_fn(*snake_case__							) for input_tensors_chunk in zip(*snake_case__							)							)
														# concatenate output at same dimension
														return torch.cat(snake_case__       ,  dim=snake_case__							)

							return forward_fn(*snake_case__							)

def        SCREAMING_SNAKE_CASE_					(						snake_case__       ,  snake_case__       ,  snake_case__       ,  snake_case__							)					->     Tuple[Set[int], torch.LongTensor]:
							lowerCAmelCase     		=		torch.ones(snake_case__       ,  snake_case__							)
							lowerCAmelCase     		=		set(snake_case__							) - already_pruned_heads  # Convert to set and remove already pruned heads
							for head in heads:
														# Compute how many pruned heads are before the head and move the index accordingly
														lowerCAmelCase     		=		head - sum(1 if h < head else 0 for h in already_pruned_heads							)
														lowerCAmelCase     		=		0
							lowerCAmelCase     		=		mask.view(-1							).contiguous().eq(1							)
							lowerCAmelCase     		=		torch.arange(len(snake_case__							)							)[mask].long()
							return heads, index

def        SCREAMING_SNAKE_CASE_					(						*snake_case__       ,  snake_case__ = None							)					->     Tuple[torch.Tensor, ...]:
							return torch.meshgrid(*snake_case__       ,  indexing=snake_case__							)






def        SCREAMING_SNAKE_CASE_					(						snake_case__							)					->     Tuple[torch.device, int, int]:
							return tensor.device, storage_ptr(snake_case__							), storage_size(snake_case__							)



