





import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation



def 							lowerCAmelCase_				(    A_  ,A_  ,A_ = False  ,A_ = 1  ,A_ = 1  ,A_ = 1_00_00  ,):
    assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"

    UpperCamelCase__:					Any												=      embedding_dim // 2
    UpperCamelCase__:					int												=      -math.log(A_) * torch.arange(
        start=0  ,end=A_  ,dtype=torch.floataa  ,device=timesteps.device)
    UpperCamelCase__:					Any												=      exponent / (half_dim - downscale_freq_shift)

    UpperCamelCase__:					Any												=      torch.exp(A_)
    UpperCamelCase__:					Union[str, Any]												=      timesteps[:, None].float() * emb[None, :]

    # scale embeddings
    UpperCamelCase__:					Dict												=      scale * emb

    # concat sine and cosine embeddings
    UpperCamelCase__:					Optional[Any]												=      torch.cat([torch.sin(A_), torch.cos(A_)]  ,dim=-1)

    # flip sine and cosine embeddings
    if flip_sin_to_cos:
        UpperCamelCase__:					Tuple												=      torch.cat([emb[:, half_dim:], emb[:, :half_dim]]  ,dim=-1)

    # zero pad
    if embedding_dim % 2 == 1:
        UpperCamelCase__:					Optional[int]												=      torch.nn.functional.pad(A_  ,(0, 1, 0, 0))
    return emb



def 							lowerCAmelCase_				(    A_  ,A_  ,A_=False  ,A_=0):
    UpperCamelCase__:					Dict												=      np.arange(A_  ,dtype=np.floataa)
    UpperCamelCase__:					List[Any]												=      np.arange(A_  ,dtype=np.floataa)
    UpperCamelCase__:					List[Any]												=      np.meshgrid(A_  ,A_)  # here w goes first
    UpperCamelCase__:					Any												=      np.stack(A_  ,axis=0)

    UpperCamelCase__:					List[str]												=      grid.reshape([2, 1, grid_size, grid_size])
    UpperCamelCase__:					Optional[int]												=      get_ad_sincos_pos_embed_from_grid(A_  ,A_)
    if cls_token and extra_tokens > 0:
        UpperCamelCase__:					int												=      np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed]  ,axis=0)
    return pos_embed



def 							lowerCAmelCase_				(    A_  ,A_):
    if embed_dim % 2 != 0:
        raise ValueError("embed_dim must be divisible by 2")

    # use half of dimensions to encode grid_h
    UpperCamelCase__:					Dict												=      get_ad_sincos_pos_embed_from_grid(embed_dim // 2  ,grid[0])  # (H*W, D/2)
    UpperCamelCase__:					str												=      get_ad_sincos_pos_embed_from_grid(embed_dim // 2  ,grid[1])  # (H*W, D/2)

    UpperCamelCase__:					str												=      np.concatenate([emb_h, emb_w]  ,axis=1)  # (H*W, D)
    return emb



def 							lowerCAmelCase_				(    A_  ,A_):
    if embed_dim % 2 != 0:
        raise ValueError("embed_dim must be divisible by 2")

    UpperCamelCase__:					Any												=      np.arange(embed_dim // 2  ,dtype=np.floataa)
    omega /= embed_dim / 2.0
    UpperCamelCase__:					Any												=      1.0 / 1_00_00**omega  # (D/2,)

    UpperCamelCase__:					Any												=      pos.reshape(-1)  # (M,)
    UpperCamelCase__:					Any												=      np.einsum("m,d->md"  ,A_  ,A_)  # (M, D/2), outer product

    UpperCamelCase__:					Optional[int]												=      np.sin(A_)  # (M, D/2)
    UpperCamelCase__:					Any												=      np.cos(A_)  # (M, D/2)

    UpperCamelCase__:					Optional[int]												=      np.concatenate([emb_sin, emb_cos]  ,axis=1)  # (M, D)
    return emb




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	List[str]					,  __lowerCamelCase:	Any=224					,  __lowerCamelCase:	int=224					,  __lowerCamelCase:	str=16					,  __lowerCamelCase:	List[str]=3					,  __lowerCamelCase:	List[str]=768					,  __lowerCamelCase:	int=False					,  __lowerCamelCase:	Optional[int]=True					,  __lowerCamelCase:	Union[str, Any]=True					,  ):
          '''simple docstring'''


          super().__init__()

          UpperCamelCase__:					str												=      (height // patch_size) * (width // patch_size)
          UpperCamelCase__:					str												=      flatten
          UpperCamelCase__:					Tuple												=      layer_norm

          UpperCamelCase__:					Union[str, Any]												=      nn.Convad(
              __lowerCamelCase					,  __lowerCamelCase					,  kernel_size=(patch_size, patch_size)					,  stride=__lowerCamelCase					,  bias=__lowerCamelCase			)
          if layer_norm:
              UpperCamelCase__:					List[Any]												=      nn.LayerNorm(__lowerCamelCase					,  elementwise_affine=__lowerCamelCase					,  eps=1e-6			)
          else:
              UpperCamelCase__:					Optional[int]												=      None

          UpperCamelCase__:					str												=      get_ad_sincos_pos_embed(__lowerCamelCase					,  int(num_patches**0.5			)			)
          self.register_buffer("pos_embed"					,  torch.from_numpy(__lowerCamelCase			).float().unsqueeze(0			)					,  persistent=__lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	Union[str, Any]					,  __lowerCamelCase:	Tuple			):
          '''simple docstring'''


          UpperCamelCase__:					Union[str, Any]												=      self.proj(__lowerCamelCase			)
          if self.flatten:
              UpperCamelCase__:					Any												=      latent.flatten(2			).transpose(1					,  2			)  # BCHW -> BNC
          if self.layer_norm:
              UpperCamelCase__:					Any												=      self.norm(__lowerCamelCase			)
          return latent + self.pos_embed




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Union[str, Any]					,  __lowerCamelCase:	int					,  __lowerCamelCase:	int					,  __lowerCamelCase:	str = "silu"					,  __lowerCamelCase:	int = None					,  __lowerCamelCase:	Optional[str] = None					,  __lowerCamelCase:	Optional[int]=None					,  ):
          '''simple docstring'''


          super().__init__()

          UpperCamelCase__:					Dict												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)

          if cond_proj_dim is not None:
              UpperCamelCase__:					Tuple												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase					,  bias=__lowerCamelCase			)
          else:
              UpperCamelCase__:					str												=      None

          UpperCamelCase__:					List[Any]												=      get_activation(__lowerCamelCase			)

          if out_dim is not None:
              UpperCamelCase__:					Union[str, Any]												=      out_dim
          else:
              UpperCamelCase__:					Optional[int]												=      time_embed_dim
          UpperCamelCase__:					str												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)

          if post_act_fn is None:
              UpperCamelCase__:					Optional[int]												=      None
          else:
              UpperCamelCase__:					List[Any]												=      get_activation(__lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	List[Any]					,  __lowerCamelCase:	int					,  __lowerCamelCase:	Union[str, Any]=None			):
          '''simple docstring'''


          if condition is not None:
              UpperCamelCase__:					int												=      sample + self.cond_proj(__lowerCamelCase			)
          UpperCamelCase__:					Tuple												=      self.linear_a(__lowerCamelCase			)

          if self.act is not None:
              UpperCamelCase__:					Optional[Any]												=      self.act(__lowerCamelCase			)

          UpperCamelCase__:					Optional[int]												=      self.linear_a(__lowerCamelCase			)

          if self.post_act is not None:
              UpperCamelCase__:					Optional[Any]												=      self.post_act(__lowerCamelCase			)
          return sample




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Union[str, Any]					,  __lowerCamelCase:	int					,  __lowerCamelCase:	bool					,  __lowerCamelCase:	float			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					int												=      num_channels
          UpperCamelCase__:					Tuple												=      flip_sin_to_cos
          UpperCamelCase__:					Tuple												=      downscale_freq_shift






      def 	UpperCAmelCase_   (						self:	Optional[int]					,  __lowerCamelCase:	Optional[Any]			):
          '''simple docstring'''


          UpperCamelCase__:					int												=      get_timestep_embedding(
              __lowerCamelCase					,  self.num_channels					,  flip_sin_to_cos=self.flip_sin_to_cos					,  downscale_freq_shift=self.downscale_freq_shift					,  )
          return t_emb




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Any					,  __lowerCamelCase:	int = 256					,  __lowerCamelCase:	float = 1.0					,  __lowerCamelCase:	int=True					,  __lowerCamelCase:	str=True					,  __lowerCamelCase:	Optional[Any]=False			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					Tuple												=      nn.Parameter(torch.randn(__lowerCamelCase			) * scale					,  requires_grad=__lowerCamelCase			)
          UpperCamelCase__:					Optional[Any]												=      log
          UpperCamelCase__:					Dict												=      flip_sin_to_cos

          if set_W_to_weight:
              # to delete later
              UpperCamelCase__:					Union[str, Any]												=      nn.Parameter(torch.randn(__lowerCamelCase			) * scale					,  requires_grad=__lowerCamelCase			)

              UpperCamelCase__:					Dict												=      self.W






      def 	UpperCAmelCase_   (						self:	Union[str, Any]					,  __lowerCamelCase:	Dict			):
          '''simple docstring'''


          if self.log:
              UpperCamelCase__:					Union[str, Any]												=      torch.log(__lowerCamelCase			)

          UpperCamelCase__:					Union[str, Any]												=      x[:, None] * self.weight[None, :] * 2 * np.pi

          if self.flip_sin_to_cos:
              UpperCamelCase__:					Dict												=      torch.cat([torch.cos(__lowerCamelCase			), torch.sin(__lowerCamelCase			)]					,  dim=-1			)
          else:
              UpperCamelCase__:					Optional[int]												=      torch.cat([torch.sin(__lowerCamelCase			), torch.cos(__lowerCamelCase			)]					,  dim=-1			)
          return out




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Tuple					,  __lowerCamelCase:	int					,  __lowerCamelCase:	int					,  __lowerCamelCase:	int					,  __lowerCamelCase:	int					,  ):
          '''simple docstring'''


          super().__init__()

          UpperCamelCase__:					Union[str, Any]												=      height
          UpperCamelCase__:					Dict												=      width
          UpperCamelCase__:					str												=      num_embed
          UpperCamelCase__:					Any												=      embed_dim

          UpperCamelCase__:					int												=      nn.Embedding(self.num_embed					,  __lowerCamelCase			)
          UpperCamelCase__:					Tuple												=      nn.Embedding(self.height					,  __lowerCamelCase			)
          UpperCamelCase__:					Any												=      nn.Embedding(self.width					,  __lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	Any					,  __lowerCamelCase:	Tuple			):
          '''simple docstring'''


          UpperCamelCase__:					List[str]												=      self.emb(__lowerCamelCase			)

          UpperCamelCase__:					Union[str, Any]												=      self.height_emb(torch.arange(self.height					,  device=index.device			).view(1					,  self.height			)			)

          # 1 x H x D -> 1 x H x 1 x D
          UpperCamelCase__:					int												=      height_emb.unsqueeze(2			)

          UpperCamelCase__:					Dict												=      self.width_emb(torch.arange(self.width					,  device=index.device			).view(1					,  self.width			)			)

          # 1 x W x D -> 1 x 1 x W x D
          UpperCamelCase__:					int												=      width_emb.unsqueeze(1			)

          UpperCamelCase__:					Tuple												=      height_emb + width_emb

          # 1 x H x W x D -> 1 x L xD
          UpperCamelCase__:					Optional[int]												=      pos_emb.view(1					,  self.height * self.width					,  -1			)

          UpperCamelCase__:					Dict												=      emb + pos_emb[:, : emb.shape[1], :]

          return emb




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	str					,  __lowerCamelCase:	str					,  __lowerCamelCase:	Any					,  __lowerCamelCase:	Any			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					Dict												=      dropout_prob > 0
          UpperCamelCase__:					List[str]												=      nn.Embedding(num_classes + use_cfg_embedding					,  __lowerCamelCase			)
          UpperCamelCase__:					Tuple												=      num_classes
          UpperCamelCase__:					Optional[int]												=      dropout_prob






      def 	UpperCAmelCase_   (						self:	Optional[int]					,  __lowerCamelCase:	List[str]					,  __lowerCamelCase:	str=None			):
          '''simple docstring'''


          if force_drop_ids is None:
              UpperCamelCase__:					Optional[int]												=      torch.rand(labels.shape[0]					,  device=labels.device			) < self.dropout_prob
          else:
              UpperCamelCase__:					List[Any]												=      torch.tensor(force_drop_ids == 1			)
          UpperCamelCase__:					Optional[Any]												=      torch.where(__lowerCamelCase					,  self.num_classes					,  __lowerCamelCase			)
          return labels






      def 	UpperCAmelCase_   (						self:	Dict					,  __lowerCamelCase:	torch.LongTensor					,  __lowerCamelCase:	List[Any]=None			):
          '''simple docstring'''


          UpperCamelCase__:					str												=      self.dropout_prob > 0
          if (self.training and use_dropout) or (force_drop_ids is not None):
              UpperCamelCase__:					List[Any]												=      self.token_drop(__lowerCamelCase					,  __lowerCamelCase			)
          UpperCamelCase__:					Optional[int]												=      self.embedding_table(__lowerCamelCase			)
          return embeddings




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	List[str]					,  __lowerCamelCase:	int = 1024					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 10					,  ):
          '''simple docstring'''


          super().__init__()

          UpperCamelCase__:					Optional[Any]												=      num_image_text_embeds
          UpperCamelCase__:					Any												=      nn.Linear(__lowerCamelCase					,  self.num_image_text_embeds * cross_attention_dim			)
          UpperCamelCase__:					Optional[int]												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	Tuple					,  __lowerCamelCase:	torch.FloatTensor					,  __lowerCamelCase:	torch.FloatTensor			):
          '''simple docstring'''


          UpperCamelCase__:					Tuple												=      text_embeds.shape[0]

          # image
          UpperCamelCase__:					Optional[Any]												=      self.image_embeds(__lowerCamelCase			)
          UpperCamelCase__:					str												=      image_text_embeds.reshape(__lowerCamelCase					,  self.num_image_text_embeds					,  -1			)

          # text
          UpperCamelCase__:					Optional[int]												=      self.text_proj(__lowerCamelCase			)

          return torch.cat([image_text_embeds, text_embeds]					,  dim=1			)




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	List[str]					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 32					,  ):
          '''simple docstring'''


          super().__init__()

          UpperCamelCase__:					List[Any]												=      num_image_text_embeds
          UpperCamelCase__:					List[Any]												=      nn.Linear(__lowerCamelCase					,  self.num_image_text_embeds * cross_attention_dim			)
          UpperCamelCase__:					Tuple												=      nn.LayerNorm(__lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	Any					,  __lowerCamelCase:	torch.FloatTensor			):
          '''simple docstring'''


          UpperCamelCase__:					int												=      image_embeds.shape[0]

          # image
          UpperCamelCase__:					Optional[Any]												=      self.image_embeds(__lowerCamelCase			)
          UpperCamelCase__:					Union[str, Any]												=      image_embeds.reshape(__lowerCamelCase					,  self.num_image_text_embeds					,  -1			)
          UpperCamelCase__:					Dict												=      self.norm(__lowerCamelCase			)
          return image_embeds




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Union[str, Any]					,  __lowerCamelCase:	Any					,  __lowerCamelCase:	Tuple					,  __lowerCamelCase:	List[Any]=0.1			):
          '''simple docstring'''


          super().__init__()

          UpperCamelCase__:					List[Any]												=      Timesteps(num_channels=256					,  flip_sin_to_cos=__lowerCamelCase					,  downscale_freq_shift=1			)
          UpperCamelCase__:					List[str]												=      TimestepEmbedding(in_channels=256					,  time_embed_dim=__lowerCamelCase			)
          UpperCamelCase__:					Optional[Any]												=      LabelEmbedding(__lowerCamelCase					,  __lowerCamelCase					,  __lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	List[Any]					,  __lowerCamelCase:	Dict					,  __lowerCamelCase:	int					,  __lowerCamelCase:	Any=None			):
          '''simple docstring'''


          UpperCamelCase__:					str												=      self.time_proj(__lowerCamelCase			)
          UpperCamelCase__:					Any												=      self.timestep_embedder(timesteps_proj.to(dtype=__lowerCamelCase			)			)  # (N, D)

          UpperCamelCase__:					Optional[int]												=      self.class_embedder(__lowerCamelCase			)  # (N, D)

          UpperCamelCase__:					int												=      timesteps_emb + class_labels  # (N, D)

          return conditioning




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Optional[int]					,  __lowerCamelCase:	int					,  __lowerCamelCase:	int					,  __lowerCamelCase:	int = 64			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					str												=      nn.LayerNorm(__lowerCamelCase			)
          UpperCamelCase__:					int												=      AttentionPooling(__lowerCamelCase					,  __lowerCamelCase			)
          UpperCamelCase__:					str												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)
          UpperCamelCase__:					str												=      nn.LayerNorm(__lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	int					,  __lowerCamelCase:	Union[str, Any]			):
          '''simple docstring'''


          UpperCamelCase__:					Dict												=      self.norma(__lowerCamelCase			)
          UpperCamelCase__:					Optional[int]												=      self.pool(__lowerCamelCase			)
          UpperCamelCase__:					Union[str, Any]												=      self.proj(__lowerCamelCase			)
          UpperCamelCase__:					Optional[int]												=      self.norma(__lowerCamelCase			)
          return hidden_states




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Union[str, Any]					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 1536			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					Union[str, Any]												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)
          UpperCamelCase__:					Tuple												=      nn.LayerNorm(__lowerCamelCase			)
          UpperCamelCase__:					Dict												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	List[str]					,  __lowerCamelCase:	torch.FloatTensor					,  __lowerCamelCase:	torch.FloatTensor			):
          '''simple docstring'''


          UpperCamelCase__:					List[Any]												=      self.text_proj(__lowerCamelCase			)
          UpperCamelCase__:					Optional[int]												=      self.text_norm(__lowerCamelCase			)

          # image
          UpperCamelCase__:					Tuple												=      self.image_proj(__lowerCamelCase			)

          return time_image_embeds + time_text_embeds




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	List[Any]					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 1536			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					int												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)
          UpperCamelCase__:					List[str]												=      nn.LayerNorm(__lowerCamelCase			)






      def 	UpperCAmelCase_   (						self:	Any					,  __lowerCamelCase:	torch.FloatTensor			):
          '''simple docstring'''


          UpperCamelCase__:					Optional[Any]												=      self.image_proj(__lowerCamelCase			)
          UpperCamelCase__:					Union[str, Any]												=      self.image_norm(__lowerCamelCase			)
          return time_image_embeds




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Union[str, Any]					,  __lowerCamelCase:	int = 768					,  __lowerCamelCase:	int = 1536			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					List[str]												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase			)
          UpperCamelCase__:					Union[str, Any]												=      nn.LayerNorm(__lowerCamelCase			)
          UpperCamelCase__:					int												=      nn.Sequential(
              nn.Convad(3					,  16					,  3					,  padding=1			)					,  nn.SiLU()					,  nn.Convad(16					,  16					,  3					,  padding=1			)					,  nn.SiLU()					,  nn.Convad(16					,  32					,  3					,  padding=1					,  stride=2			)					,  nn.SiLU()					,  nn.Convad(32					,  32					,  3					,  padding=1			)					,  nn.SiLU()					,  nn.Convad(32					,  96					,  3					,  padding=1					,  stride=2			)					,  nn.SiLU()					,  nn.Convad(96					,  96					,  3					,  padding=1			)					,  nn.SiLU()					,  nn.Convad(96					,  256					,  3					,  padding=1					,  stride=2			)					,  nn.SiLU()					,  nn.Convad(256					,  4					,  3					,  padding=1			)					,  )






      def 	UpperCAmelCase_   (						self:	Tuple					,  __lowerCamelCase:	torch.FloatTensor					,  __lowerCamelCase:	torch.FloatTensor			):
          '''simple docstring'''


          UpperCamelCase__:					Optional[Any]												=      self.image_proj(__lowerCamelCase			)
          UpperCamelCase__:					int												=      self.image_norm(__lowerCamelCase			)
          UpperCamelCase__:					Union[str, Any]												=      self.input_hint_block(__lowerCamelCase			)
          return time_image_embeds, hint




class    _a  (  nn.Module):

      """simple docstring"""


      def __init__(						self:	Union[str, Any]					,  __lowerCamelCase:	Dict					,  __lowerCamelCase:	Optional[Any]					,  __lowerCamelCase:	Any=None			):
          '''simple docstring'''


          super().__init__()
          UpperCamelCase__:					str												=      dtype
          UpperCamelCase__:					str												=      nn.Parameter(torch.randn(1					,  __lowerCamelCase			) / embed_dim**0.5			)
          UpperCamelCase__:					Optional[int]												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase					,  dtype=self.dtype			)
          UpperCamelCase__:					Tuple												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase					,  dtype=self.dtype			)
          UpperCamelCase__:					List[str]												=      nn.Linear(__lowerCamelCase					,  __lowerCamelCase					,  dtype=self.dtype			)
          UpperCamelCase__:					int												=      num_heads
          UpperCamelCase__:					List[str]												=      embed_dim // self.num_heads






      def 	UpperCAmelCase_   (						self:	Union[str, Any]					,  __lowerCamelCase:	Tuple			):
          '''simple docstring'''


          UpperCamelCase__							,				UpperCamelCase__							,				UpperCamelCase__:					List[Any]												=      x.size()

          def shape(__lowerCamelCase:	List[Any]			):
              # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
              UpperCamelCase__:					List[str]												=      x.view(__lowerCamelCase					,  -1					,  self.num_heads					,  self.dim_per_head			)
              # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
              UpperCamelCase__:					Optional[int]												=      x.transpose(1					,  2			)
              # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
              UpperCamelCase__:					Tuple												=      x.reshape(bs * self.num_heads					,  -1					,  self.dim_per_head			)
              # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
              UpperCamelCase__:					int												=      x.transpose(1					,  2			)
              return x

          UpperCamelCase__:					str												=      x.mean(dim=1					,  keepdim=__lowerCamelCase			) + self.positional_embedding.to(x.dtype			)
          UpperCamelCase__:					Union[str, Any]												=      torch.cat([class_token, x]					,  dim=1			)  # (bs, length+1, width)

          # (bs*n_heads, class_token_length, dim_per_head)
          UpperCamelCase__:					Tuple												=      shape(self.q_proj(__lowerCamelCase			)			)
          # (bs*n_heads, length+class_token_length, dim_per_head)
          UpperCamelCase__:					Tuple												=      shape(self.k_proj(__lowerCamelCase			)			)
          UpperCamelCase__:					Any												=      shape(self.v_proj(__lowerCamelCase			)			)

          # (bs*n_heads, class_token_length, length+class_token_length):
          UpperCamelCase__:					str												=      1 / math.sqrt(math.sqrt(self.dim_per_head			)			)
          UpperCamelCase__:					int												=      torch.einsum("bct,bcs->bts"					,  q * scale					,  k * scale			)  # More stable with f16 than dividing afterwards
          UpperCamelCase__:					List[str]												=      torch.softmax(weight.float()					,  dim=-1			).type(weight.dtype			)

          # (bs*n_heads, dim_per_head, class_token_length)
          UpperCamelCase__:					str												=      torch.einsum("bts,bcs->bct"					,  __lowerCamelCase					,  __lowerCamelCase			)

          # (bs, length+1, width)
          UpperCamelCase__:					Any												=      a.reshape(__lowerCamelCase					,  -1					,  1			).transpose(1					,  2			)

          return a[:, 0, :]  # cls_token








from .chunk_utils import chunk_layer
from .data_transforms import make_atomaa_masks
from .feats import atomaa_to_atomaa, frames_and_literature_positions_to_atomaa_pos, torsion_angles_to_frames
from .loss import compute_predicted_aligned_error, compute_tm
from .protein import Protein as OFProtein
from .protein import to_pdb
from .rigid_utils import Rigid, Rotation
from .tensor_utils import dict_multimap, flatten_final_dims, permute_final_dims

