



"""simple docstring"""
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
from .unet_ad_blocks import UNetMidBlockaD, get_down_block, get_up_block
@dataclass
class     lowerCamelCase  (      _lowerCAmelCase    ):






					'''simple docstring'''





					_A		:      torch.FloatTensor

class     lowerCamelCase  (      _lowerCAmelCase   ,						_lowerCAmelCase    ):






					'''simple docstring'''





					@register_to_config
					def __init__(	self:							Union[str, Any]						,					snake_case:							Optional[Union[int, Tuple[int, int]]] = None						,					snake_case:							int = 3						,					snake_case:							int = 3						,					snake_case:							bool = False						,					snake_case:							str = "positional"						,					snake_case:							int = 0						,					snake_case:							bool = True						,					snake_case:							Tuple[str] = ("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D")						,					snake_case:							Tuple[str] = ("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D")						,					snake_case:							Tuple[int] = (224, 448, 672, 896)						,					snake_case:							int = 2						,					snake_case:							float = 1						,					snake_case:							int = 1						,					snake_case:							str = "conv"						,					snake_case:							str = "conv"						,					snake_case:							str = "silu"						,					snake_case:							Optional[int] = 8						,					snake_case:							int = 32						,					snake_case:							float = 1E-5						,					snake_case:							str = "default"						,					snake_case:							bool = True						,					snake_case:							Optional[str] = None						,					snake_case:							Optional[int] = None						,					)       ->     int:
											super().__init__()

											snake_case_			:Any						=      sample_size
											snake_case_			:int						=      block_out_channels[0] * 4

											# Check inputs
											if len(snake_case						) != len(snake_case						):
																	raise ValueError(
																	    f"""Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}."""						)

											if len(snake_case						) != len(snake_case						):
																	raise ValueError(
																	    f"""Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."""						)

											# input
											snake_case_			:List[Any]						=      nn.Convad(snake_case						,					block_out_channels[0]						,					kernel_size=3						,					padding=(1, 1)						)

											# time
											if time_embedding_type == "fourier":
																	snake_case_			:Union[str, Any]						=      GaussianFourierProjection(embedding_size=block_out_channels[0]						,					scale=16						)
																	snake_case_			:str						=      2 * block_out_channels[0]
											elif time_embedding_type == "positional":
																	snake_case_			:Dict						=      Timesteps(block_out_channels[0]						,					snake_case						,					snake_case						)
																	snake_case_			:str						=      block_out_channels[0]

											snake_case_			:Union[str, Any]						=      TimestepEmbedding(snake_case						,					snake_case						)

											# class embedding
											if class_embed_type is None and num_class_embeds is not None:
																	snake_case_			:Tuple						=      nn.Embedding(snake_case						,					snake_case						)
											elif class_embed_type == "timestep":
																	snake_case_			:List[Any]						=      TimestepEmbedding(snake_case						,					snake_case						)
											elif class_embed_type == "identity":
																	snake_case_			:Optional[Any]						=      nn.Identity(snake_case						,					snake_case						)
											else:
																	snake_case_			:List[Any]						=      None

											snake_case_			:List[Any]						=      nn.ModuleList([]						)
											snake_case_			:Optional[int]						=      None
											snake_case_			:Union[str, Any]						=      nn.ModuleList([]						)

											# down
											snake_case_			:Union[str, Any]						=      block_out_channels[0]
											for i, down_block_type in enumerate(snake_case						):
																	snake_case_			:Optional[int]						=      output_channel
																	snake_case_			:Dict						=      block_out_channels[i]
																	snake_case_			:List[Any]						=      i == len(snake_case						) - 1

																	snake_case_			:Union[str, Any]						=      get_down_block(
																	    snake_case						,					num_layers=snake_case						,					in_channels=snake_case						,					out_channels=snake_case						,					temb_channels=snake_case						,					add_downsample=not is_final_block						,					resnet_eps=snake_case						,					resnet_act_fn=snake_case						,					resnet_groups=snake_case						,					attention_head_dim=attention_head_dim if attention_head_dim is not None else output_channel						,					downsample_padding=snake_case						,					resnet_time_scale_shift=snake_case						,					downsample_type=snake_case						,					)
																	self.down_blocks.append(snake_case						)

											# mid
											snake_case_			:int						=      UNetMidBlockaD(
											    in_channels=block_out_channels[-1]						,					temb_channels=snake_case						,					resnet_eps=snake_case						,					resnet_act_fn=snake_case						,					output_scale_factor=snake_case						,					resnet_time_scale_shift=snake_case						,					attention_head_dim=attention_head_dim if attention_head_dim is not None else block_out_channels[-1]						,					resnet_groups=snake_case						,					add_attention=snake_case						,					)

											# up
											snake_case_			:Optional[Any]						=      list(reversed(snake_case						)						)
											snake_case_			:Tuple						=      reversed_block_out_channels[0]
											for i, up_block_type in enumerate(snake_case						):
																	snake_case_			:List[Any]						=      output_channel
																	snake_case_			:Dict						=      reversed_block_out_channels[i]
																	snake_case_			:Tuple						=      reversed_block_out_channels[min(i + 1						,					len(snake_case						) - 1						)]

																	snake_case_			:Union[str, Any]						=      i == len(snake_case						) - 1

																	snake_case_			:str						=      get_up_block(
																	    snake_case						,					num_layers=layers_per_block + 1						,					in_channels=snake_case						,					out_channels=snake_case						,					prev_output_channel=snake_case						,					temb_channels=snake_case						,					add_upsample=not is_final_block						,					resnet_eps=snake_case						,					resnet_act_fn=snake_case						,					resnet_groups=snake_case						,					attention_head_dim=attention_head_dim if attention_head_dim is not None else output_channel						,					resnet_time_scale_shift=snake_case						,					upsample_type=snake_case						,					)
																	self.up_blocks.append(snake_case						)
																	snake_case_			:int						=      output_channel

											# out
											snake_case_			:Union[str, Any]						=      norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4						,					32						)
											snake_case_			:Optional[Any]						=      nn.GroupNorm(num_channels=block_out_channels[0]						,					num_groups=snake_case						,					eps=snake_case						)
											snake_case_			:List[Any]						=      nn.SiLU()
											snake_case_			:Optional[Any]						=      nn.Convad(block_out_channels[0]						,					snake_case						,					kernel_size=3						,					padding=1						)






					def        lowerCAmelCase_    (	self:							List[str]						,					snake_case:							torch.FloatTensor						,					snake_case:							Union[torch.Tensor, float, int]						,					snake_case:							Optional[torch.Tensor] = None						,					snake_case:							bool = True						,					)       ->     Union[UNetaDOutput, Tuple]:
											# 0. center input if necessary
											if self.config.center_input_sample:
																	snake_case_			:Tuple						=      2 * sample - 1.0

											# 1. time
											snake_case_			:Any						=      timestep
											if not torch.is_tensor(snake_case						):
																	snake_case_			:Tuple						=      torch.tensor([timesteps]						,					dtype=torch.long						,					device=sample.device						)
											elif torch.is_tensor(snake_case						) and len(timesteps.shape						) == 0:
																	snake_case_			:Dict						=      timesteps[None].to(sample.device						)

											# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
											snake_case_			:Union[str, Any]						=      timesteps * torch.ones(sample.shape[0]						,					dtype=timesteps.dtype						,					device=timesteps.device						)

											snake_case_			:Tuple						=      self.time_proj(snake_case						)

											# timesteps does not contain any weights and will always return f32 tensors
											# but time_embedding might actually be running in fp16. so we need to cast here.
											# there might be better ways to encapsulate this.
											snake_case_			:Any						=      t_emb.to(dtype=self.dtype						)
											snake_case_			:Optional[int]						=      self.time_embedding(snake_case						)

											if self.class_embedding is not None:
																	if class_labels is None:
																							raise ValueError("""class_labels should be provided when doing class conditioning"""						)

																	if self.config.class_embed_type == "timestep":
																							snake_case_			:Optional[int]						=      self.time_proj(snake_case						)

																	snake_case_			:List[str]						=      self.class_embedding(snake_case						).to(dtype=self.dtype						)
																	snake_case_			:int						=      emb + class_emb

											# 2. pre-process
											snake_case_			:Optional[Any]						=      sample
											snake_case_			:Any						=      self.conv_in(snake_case						)

											# 3. down
											snake_case_			:Union[str, Any]						=      (sample,)
											for downsample_block in self.down_blocks:
																	if hasattr(snake_case						,					"""skip_conv"""						):
																							snake_case_			:List[Any]						=      downsample_block(
																							    hidden_states=snake_case						,					temb=snake_case						,					skip_sample=snake_case						)
																	else:
																							snake_case_			:Any						=      downsample_block(hidden_states=snake_case						,					temb=snake_case						)

																	down_block_res_samples += res_samples

											# 4. mid
											snake_case_			:Optional[int]						=      self.mid_block(snake_case						,					snake_case						)

											# 5. up
											snake_case_			:Tuple						=      None
											for upsample_block in self.up_blocks:
																	snake_case_			:Dict						=      down_block_res_samples[-len(upsample_block.resnets						) :]
																	snake_case_			:List[Any]						=      down_block_res_samples[: -len(upsample_block.resnets						)]

																	if hasattr(snake_case						,					"""skip_conv"""						):
																							snake_case_			:Union[str, Any]						=      upsample_block(snake_case						,					snake_case						,					snake_case						,					snake_case						)
																	else:
																							snake_case_			:Any						=      upsample_block(snake_case						,					snake_case						,					snake_case						)

        # 6. post-process
											snake_case_			:Optional[int]						=      self.conv_norm_out(snake_case						)
											snake_case_			:Union[str, Any]						=      self.conv_act(snake_case						)
											snake_case_			:Optional[Any]						=      self.conv_out(snake_case						)

											if skip_sample is not None:
																	sample += skip_sample

											if self.config.time_embedding_type == "fourier":
																	snake_case_			:List[Any]						=      timesteps.reshape((sample.shape[0], *([1] * len(sample.shape[1:]						)))						)
																	snake_case_			:str						=      sample / timesteps

											if not return_dict:
																	return (sample,)

											return UNetaDOutput(sample=snake_case						)





"""simple docstring"""
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


__a     =    logging.get_logger(__name__)

__a     =    {
    "google/long-t5-local-base": "https://huggingface.co/google/long-t5-local-base/blob/main/config.json",
    "google/long-t5-local-large": "https://huggingface.co/google/long-t5-local-large/blob/main/config.json",
    "google/long-t5-tglobal-base": "https://huggingface.co/google/long-t5-tglobal-base/blob/main/config.json",
    "google/long-t5-tglobal-large": "https://huggingface.co/google/long-t5-tglobal-large/blob/main/config.json",
}
class     lowerCamelCase  (      _lowerCAmelCase    ):






					'''simple docstring'''





					_A		:      List[Any]						=					"""longt5"""
					_A		:      Dict						=					["""past_key_values"""]
					_A		:      int						=					{"""hidden_size""": """d_model""", """num_attention_heads""": """num_heads""", """num_hidden_layers""": """num_layers"""}






					def __init__(	self:							str						,					snake_case:							List[Any]=32_128						,					snake_case:							Optional[int]=512						,					snake_case:							Union[str, Any]=64						,					snake_case:							str=2_048						,					snake_case:							Any=6						,					snake_case:							Optional[Any]=None						,					snake_case:							str=8						,					snake_case:							Tuple=127						,					snake_case:							Any=16						,					snake_case:							int=32						,					snake_case:							int=128						,					snake_case:							int=0.1						,					snake_case:							Optional[int]=1E-6						,					snake_case:							List[Any]=1.0						,					snake_case:							Optional[int]="relu"						,					snake_case:							str=True						,					snake_case:							Optional[int]="local"						,					snake_case:							Any=True						,					snake_case:							Tuple=0						,					snake_case:							Any=1						,					**snake_case:							Any						,					)       ->     List[str]:
											snake_case_			:Any						=      vocab_size
											snake_case_			:List[Any]						=      d_model
											snake_case_			:Any						=      d_kv
											snake_case_			:int						=      d_ff
											snake_case_			:Tuple						=      num_layers
											# default = symmetry
											snake_case_			:str						=      num_decoder_layers if num_decoder_layers is not None else self.num_layers
											snake_case_			:Any						=      num_heads
											snake_case_			:str						=      local_radius
											snake_case_			:Optional[int]						=      global_block_size
											snake_case_			:Union[str, Any]						=      relative_attention_num_buckets
											snake_case_			:List[str]						=      relative_attention_max_distance
											snake_case_			:Dict						=      dropout_rate
											snake_case_			:str						=      layer_norm_epsilon
											snake_case_			:str						=      initializer_factor
											snake_case_			:Any						=      feed_forward_proj
											snake_case_			:List[str]						=      encoder_attention_type
											snake_case_			:List[str]						=      use_cache

											snake_case_			:Tuple						=      self.feed_forward_proj.split("""-"""						)
											snake_case_			:Any						=      act_info[-1]
											snake_case_			:Union[str, Any]						=      act_info[0] == """gated"""

											if len(snake_case						) > 1 and act_info[0] != "gated" or len(snake_case						) > 2:
																	raise ValueError(
																	    f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
																	    """Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. """
																	    """'gated-gelu' or 'relu'"""						)

											# for backwards compatibility
											if feed_forward_proj == "gated-gelu":
																	snake_case_			:Optional[int]						=      """gelu_new"""

											super().__init__(
											    pad_token_id=snake_case						,					eos_token_id=snake_case						,					is_encoder_decoder=snake_case						,					**snake_case						,					)

class     lowerCamelCase  (      _lowerCAmelCase    ):






					'''simple docstring'''





					@property
					def        lowerCAmelCase_    (	self:							Dict						)       ->     Mapping[str, Mapping[int, str]]:
											snake_case_			:Dict						=      {
											    """input_ids""": {0: """batch""", 1: """encoder_sequence"""},
											    """attention_mask""": {0: """batch""", 1: """encoder_sequence"""},
											}
											if self.use_past:
																	snake_case_			:List[Any]						=      """past_encoder_sequence + sequence"""
																	snake_case_			:List[Any]						=      {0: """batch"""}
																	snake_case_			:List[Any]						=      {0: """batch""", 1: """past_decoder_sequence + sequence"""}
											else:
																	snake_case_			:List[str]						=      {0: """batch""", 1: """decoder_sequence"""}
																	snake_case_			:List[Any]						=      {0: """batch""", 1: """decoder_sequence"""}

											if self.use_past:
																	self.fill_with_past_key_values_(snake_case						,					direction="""inputs"""						)

											return common_inputs






					@property
					def        lowerCAmelCase_    (	self:							List[str]						)       ->     int:
											return 13
