



'''simple docstring'''




from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin






@dataclass
class 						_UpperCamelCase  (							lowerCamelCase__   ):


			'''simple docstring'''

			_A				:		torch.FloatTensor






class 						_UpperCamelCase  (							lowerCamelCase__   , lowerCamelCase__   ):


			'''simple docstring'''

			@register_to_config
			def __init__(							self					:			Optional[int]							,					lowerCAmelCase__					:			int = 1_6							,					lowerCAmelCase__					:			int = 8_8							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			int = 1							,					lowerCAmelCase__					:			float = 0.0							,					lowerCAmelCase__					:			int = 3_2							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			bool = False							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			str = "geglu"							,					lowerCAmelCase__					:			Optional[int] = None							,					lowerCAmelCase__					:			bool = False							,					lowerCAmelCase__					:			bool = False							,					lowerCAmelCase__					:			bool = False							,					lowerCAmelCase__					:			str = "layer_norm"							,					lowerCAmelCase__					:			bool = True							,					):





					"""simple docstring"""






					super().__init__()
					__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		use_linear_projection
					__SCREAMING_SNAKE_CASE	:		str   	=		num_attention_heads
					__SCREAMING_SNAKE_CASE	:		Tuple   	=		attention_head_dim
					__SCREAMING_SNAKE_CASE	:		str   	=		num_attention_heads * attention_head_dim

					# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
					# Define whether input is continuous or discrete depending on configuration
					__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		(in_channels is not None) and (patch_size is None)
					__SCREAMING_SNAKE_CASE	:		List[Any]   	=		num_vector_embeds is not None
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		in_channels is not None and patch_size is not None

					if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
							__SCREAMING_SNAKE_CASE	:		str   	=		(
							    F"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"
							    """ incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."""
							    """ Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"""
							    """ results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"""
							    """ would be very nice if you could open a Pull request for the `transformer/config.json` file"""
							)
							deprecate("""norm_type!=num_embeds_ada_norm"""							,					"""1.0.0"""							,					lowerCAmelCase__							,					standard_warn=lowerCAmelCase__						)
							__SCREAMING_SNAKE_CASE	:		Dict   	=		"""ada_norm"""

					if self.is_input_continuous and self.is_input_vectorized:
							raise ValueError(
							    F"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"
							    """ sure that either `in_channels` or `num_vector_embeds` is None."""						)
					elif self.is_input_vectorized and self.is_input_patches:
							raise ValueError(
							    F"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"
							    """ sure that either `num_vector_embeds` or `num_patches` is None."""						)
					elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
							raise ValueError(
							    F"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"
							    F" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."						)

					# 2. Define input layers
					if self.is_input_continuous:
							__SCREAMING_SNAKE_CASE	:		Any   	=		in_channels

							__SCREAMING_SNAKE_CASE	:		List[str]   	=		torch.nn.GroupNorm(num_groups=lowerCAmelCase__							,					num_channels=lowerCAmelCase__							,					eps=1E-6							,					affine=lowerCAmelCase__						)
							if use_linear_projection:
									__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		nn.Linear(lowerCAmelCase__							,					lowerCAmelCase__						)
							else:
									__SCREAMING_SNAKE_CASE	:		int   	=		nn.Convad(lowerCAmelCase__							,					lowerCAmelCase__							,					kernel_size=1							,					stride=1							,					padding=0						)
					elif self.is_input_vectorized:
							assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
							assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

							__SCREAMING_SNAKE_CASE	:		Tuple   	=		sample_size
							__SCREAMING_SNAKE_CASE	:		str   	=		sample_size
							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		num_vector_embeds
							__SCREAMING_SNAKE_CASE	:		int   	=		self.height * self.width

							__SCREAMING_SNAKE_CASE	:		Tuple   	=		ImagePositionalEmbeddings(
							    num_embed=lowerCAmelCase__							,					embed_dim=lowerCAmelCase__							,					height=self.height							,					width=self.width						)
					elif self.is_input_patches:
							assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

							__SCREAMING_SNAKE_CASE	:		List[Any]   	=		sample_size
							__SCREAMING_SNAKE_CASE	:		List[str]   	=		sample_size

							__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		patch_size
							__SCREAMING_SNAKE_CASE	:		str   	=		PatchEmbed(
							    height=lowerCAmelCase__							,					width=lowerCAmelCase__							,					patch_size=lowerCAmelCase__							,					in_channels=lowerCAmelCase__							,					embed_dim=lowerCAmelCase__							,					)

					# 3. Define transformers blocks
					__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		nn.ModuleList(
					    [
					        BasicTransformerBlock(
					            lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					dropout=lowerCAmelCase__							,					cross_attention_dim=lowerCAmelCase__							,					activation_fn=lowerCAmelCase__							,					num_embeds_ada_norm=lowerCAmelCase__							,					attention_bias=lowerCAmelCase__							,					only_cross_attention=lowerCAmelCase__							,					upcast_attention=lowerCAmelCase__							,					norm_type=lowerCAmelCase__							,					norm_elementwise_affine=lowerCAmelCase__							,					)
					        for d in range(lowerCAmelCase__						)
					    ]						)

					# 4. Define output layers
					__SCREAMING_SNAKE_CASE	:		Dict   	=		in_channels if out_channels is None else out_channels
					if self.is_input_continuous:
							# TODO: should use out_channels for continuous projections
							if use_linear_projection:
									__SCREAMING_SNAKE_CASE	:		List[str]   	=		nn.Linear(lowerCAmelCase__							,					lowerCAmelCase__						)
							else:
									__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		nn.Convad(lowerCAmelCase__							,					lowerCAmelCase__							,					kernel_size=1							,					stride=1							,					padding=0						)
					elif self.is_input_vectorized:
							__SCREAMING_SNAKE_CASE	:		int   	=		nn.LayerNorm(lowerCAmelCase__						)
							__SCREAMING_SNAKE_CASE	:		int   	=		nn.Linear(lowerCAmelCase__							,					self.num_vector_embeds - 1						)
					elif self.is_input_patches:
							__SCREAMING_SNAKE_CASE	:		Dict   	=		nn.LayerNorm(lowerCAmelCase__							,					elementwise_affine=lowerCAmelCase__							,					eps=1E-6						)
							__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		nn.Linear(lowerCAmelCase__							,					2 * inner_dim						)
							__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		nn.Linear(lowerCAmelCase__							,					patch_size * patch_size * self.out_channels						)



			def UpperCamelCase__ (							self					:			List[str]							,					lowerCAmelCase__					:			torch.Tensor							,					lowerCAmelCase__					:			Optional[torch.Tensor] = None							,					lowerCAmelCase__					:			Optional[torch.LongTensor] = None							,					lowerCAmelCase__					:			Optional[torch.LongTensor] = None							,					lowerCAmelCase__					:			Dict[str, Any] = None							,					lowerCAmelCase__					:			Optional[torch.Tensor] = None							,					lowerCAmelCase__					:			Optional[torch.Tensor] = None							,					lowerCAmelCase__					:			bool = True							,					):





					"""simple docstring"""






					if attention_mask is not None and attention_mask.ndim == 2:
							# assume that mask is expressed as:
							#   (1 = keep,      0 = discard)
							# convert mask into a bias that can be added to attention scores:
							#       (keep = +0,     discard = -10000.0)
							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		(1 - attention_mask.to(hidden_states.dtype						)) * -1_00_00.0
							__SCREAMING_SNAKE_CASE	:		Tuple   	=		attention_mask.unsqueeze(1						)

					# convert encoder_attention_mask to a bias the same way we do for attention_mask
					if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
							__SCREAMING_SNAKE_CASE	:		Any   	=		(1 - encoder_attention_mask.to(hidden_states.dtype						)) * -1_00_00.0
							__SCREAMING_SNAKE_CASE	:		Dict   	=		encoder_attention_mask.unsqueeze(1						)

					# 1. Input
					if self.is_input_continuous:
							__SCREAMING_SNAKE_CASE	,						__SCREAMING_SNAKE_CASE	,						__SCREAMING_SNAKE_CASE	,						__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		hidden_states.shape
							__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		hidden_states

							__SCREAMING_SNAKE_CASE	:		List[Any]   	=		self.norm(lowerCAmelCase__						)
							if not self.use_linear_projection:
									__SCREAMING_SNAKE_CASE	:		Any   	=		self.proj_in(lowerCAmelCase__						)
									__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		hidden_states.shape[1]
									__SCREAMING_SNAKE_CASE	:		int   	=		hidden_states.permute(0							,					2							,					3							,					1						).reshape(lowerCAmelCase__							,					height * width							,					lowerCAmelCase__						)
							else:
									__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		hidden_states.shape[1]
									__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		hidden_states.permute(0							,					2							,					3							,					1						).reshape(lowerCAmelCase__							,					height * width							,					lowerCAmelCase__						)
									__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		self.proj_in(lowerCAmelCase__						)
					elif self.is_input_vectorized:
							__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		self.latent_image_embedding(lowerCAmelCase__						)
					elif self.is_input_patches:
							__SCREAMING_SNAKE_CASE	:		Tuple   	=		self.pos_embed(lowerCAmelCase__						)

					# 2. Blocks
					for block in self.transformer_blocks:
							__SCREAMING_SNAKE_CASE	:		int   	=		block(
							    lowerCAmelCase__							,					attention_mask=lowerCAmelCase__							,					encoder_hidden_states=lowerCAmelCase__							,					encoder_attention_mask=lowerCAmelCase__							,					timestep=lowerCAmelCase__							,					cross_attention_kwargs=lowerCAmelCase__							,					class_labels=lowerCAmelCase__							,					)

					# 3. Output
					if self.is_input_continuous:
							if not self.use_linear_projection:
									__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		hidden_states.reshape(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__						).permute(0							,					3							,					1							,					2						).contiguous()
									__SCREAMING_SNAKE_CASE	:		List[Any]   	=		self.proj_out(lowerCAmelCase__						)
							else:
									__SCREAMING_SNAKE_CASE	:		int   	=		self.proj_out(lowerCAmelCase__						)
									__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		hidden_states.reshape(lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__							,					lowerCAmelCase__						).permute(0							,					3							,					1							,					2						).contiguous()

							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		hidden_states + residual
					elif self.is_input_vectorized:
							__SCREAMING_SNAKE_CASE	:		Any   	=		self.norm_out(lowerCAmelCase__						)
							__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		self.out(lowerCAmelCase__						)
							# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
							__SCREAMING_SNAKE_CASE	:		int   	=		logits.permute(0							,					2							,					1						)

							# log(p(x_0))
							__SCREAMING_SNAKE_CASE	:		Any   	=		F.log_softmax(logits.double()							,					dim=1						).float()
					elif self.is_input_patches:
							# TODO: cleanup!
							__SCREAMING_SNAKE_CASE	:		Optional[Any]   	=		self.transformer_blocks[0].norma.emb(
							    lowerCAmelCase__							,					lowerCAmelCase__							,					hidden_dtype=hidden_states.dtype						)
							__SCREAMING_SNAKE_CASE	,						__SCREAMING_SNAKE_CASE	:		List[str]   	=		self.proj_out_a(F.silu(lowerCAmelCase__						)						).chunk(2							,					dim=1						)
							__SCREAMING_SNAKE_CASE	:		str   	=		self.norm_out(lowerCAmelCase__						) * (1 + scale[:, None]) + shift[:, None]
							__SCREAMING_SNAKE_CASE	:		Any   	=		self.proj_out_a(lowerCAmelCase__						)

							# unpatchify
							__SCREAMING_SNAKE_CASE	:		Optional[int]   	=		int(hidden_states.shape[1] ** 0.5						)
							__SCREAMING_SNAKE_CASE	:		List[Any]   	=		hidden_states.reshape(
							    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)						)
							__SCREAMING_SNAKE_CASE	:		List[Any]   	=		torch.einsum("""nhwpqc->nchpwq"""							,					lowerCAmelCase__						)
							__SCREAMING_SNAKE_CASE	:		Any   	=		hidden_states.reshape(
							    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)						)

					if not return_dict:
							return (output,)

					return TransformeraDModelOutput(sample=lowerCAmelCase__						)




'''simple docstring'''




UpperCamelCase__	:     List[Any]    =			10_00


def     lowerCAmelCase_       (     _lowerCamelCase:       float ):
		return x * x


def     lowerCAmelCase_       (     _lowerCamelCase:       Any	,				_lowerCamelCase:       float	,				_lowerCamelCase:       float	,				_lowerCamelCase:       int = 4 ):
		assert callable(
		    _lowerCamelCase ), F"the function(object) passed should be callable your input : {function}"
		assert isinstance(_lowerCamelCase	,				(float, int) ), F"a should be float or integer your input : {a}"
		assert isinstance(function(_lowerCamelCase )	,				(float, int) ), (
		    "the function should return integer or float return type of your function, "
		    F"{type(_lowerCamelCase )}"
		)
		assert isinstance(_lowerCamelCase	,				(float, int) ), F"b should be float or integer your input : {b}"
		assert (
		    isinstance(_lowerCamelCase	,				_lowerCamelCase ) and precision > 0
		), F"precision should be positive integer your input : {precision}"

		# just applying the formula of simpson for approximate integration written in
		# mentioned article in first comment of this file and above this function

		__SCREAMING_SNAKE_CASE	:		List[Any]   	=		(b - a) / N_STEPS
		__SCREAMING_SNAKE_CASE	:		List[Any]   	=		function(_lowerCamelCase ) + function(_lowerCamelCase )

		for i in range(1	,				_lowerCamelCase ):
				__SCREAMING_SNAKE_CASE	:		Union[str, Any]   	=		a + h * i
				result += function(_lowerCamelCase ) * (4 if i % 2 else 2)

		result *= h / 3
		return round(_lowerCamelCase	,				_lowerCamelCase )


if __name__ == "__main__":
						import doctest

						doctest.testmod()