"""NCN recurrent workspace: shared semantic computation and dense feature voters. No CUDA initialization, downloads, training, or filesystem mutation on import. The parameter cap counts unique learned weights, including the tied vocabulary. The EMA vocabulary is a training-only teacher buffer, not an inference parameter. """ from __future__ import annotations import math from dataclasses import asdict, dataclass import torch from torch import nn from torch.nn import functional as F from torch.utils.checkpoint import checkpoint from inference_state import AttentionState, prepare_state @dataclass class WorkspaceConfig: vocab_size: int = 4096 eos_id: int = 0 context: int = 512 width: int = 256 heads: int = 8 kv_heads: int = 2 ffn_width: int = 896 core_layers: int = 6 passes: int = 2 max_passes: int = 4 experts: int = 5 expert_width: int = 256 refiner_width: int = 512 controller_width: int = 64 compass_horizons: tuple[int, ...] = (8, 32, 64) rope_theta: float = 10000.0 dropout: float = 0.0 route_floor: float = 0.001 # total floor mass; NOT per expert compass_weight: float = 0.1 prediction_weight: float = 0.02 control_weight: float = 0.0001 recovery_weight: float = 0.0 # zero is the unchanged-control training objective recovery_radius: float = 1.0 intermediate_weight: float = 0.0 head_chunk: int = 2048 parameter_cap: int = 10000000 def __post_init__(self): self.compass_horizons = tuple(self.compass_horizons) if self.width % self.heads or self.heads % self.kv_heads: raise ValueError("width/heads and heads/kv_heads must divide exactly") if (self.width // self.heads) % 2: raise ValueError("RoPE requires even head dimension") if len(self.compass_horizons) != 3 or min(self.compass_horizons) < 1: raise ValueError("exactly three positive Compass horizons required") if self.context <= max(self.compass_horizons): raise ValueError("context must exceed the longest Compass horizon") if not 1 <= self.passes <= self.max_passes or self.experts < 2: raise ValueError("invalid pass/expert count") if not 0 <= self.route_floor < 1 or self.head_chunk < 1: raise ValueError("invalid floor/head chunk") if not 0 <= self.dropout < 1 or not 0 <= self.eos_id < self.vocab_size: raise ValueError("invalid dropout/EOS") if not math.isfinite(self.recovery_weight) or self.recovery_weight < 0 or not math.isfinite(self.recovery_radius) or self.recovery_radius <= 0: raise ValueError("invalid targeted recovery settings") class RMSNorm(nn.Module): def __init__(self, width: int, affine: bool = True): super().__init__() self.weight = nn.Parameter(torch.ones(width)) if affine else None def forward(self, x): y = (x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6)).to(x.dtype) return y if self.weight is None else y * self.weight.to(y.dtype) def log_control(raw, limit, relative=False): # Center across channels only. Never center across tokens/batch: that would # let future tokens influence the causal forward computation. if relative: raw = raw - raw.mean(-1, keepdim=True) out = limit * torch.tanh(raw / 2) if relative: out = out - out.mean(-1, keepdim=True) return out class ControlHead(nn.Module): """Separate baseline and contextual correction for each physical control.""" def __init__(self, cfg): super().__init__() self.heads, self.width = cfg.heads, cfg.width size = 2 * cfg.heads + cfg.width + 1 self.baseline = nn.Parameter(torch.zeros(size)) self.contextual = nn.Linear(cfg.controller_width, size, bias=False) def forward(self, state, disabled=frozenset()): raw = self.contextual(state) baseline = self.baseline.expand_as(raw) sizes = (self.heads, self.heads, self.width, 1) names = ("precision", "pre_gain", "post_gain", "ffn_gain") limits = (math.log(2), .35, .35, .4) values = {} for name, b, r, limit in zip(names, baseline.split(sizes, -1), raw.split(sizes, -1), limits): relative = name in {"pre_gain", "post_gain"} base = log_control(b, 2.0 if name == "precision" else .7, relative) delta = log_control(r, limit, relative) if name in disabled or "all_modulation" in disabled: delta = torch.zeros_like(delta) values[name] = torch.exp(base + delta) return values, raw.float().square().mean() class Attention(nn.Module): def __init__(self, cfg): super().__init__() self.heads, self.kv_heads = cfg.heads, cfg.kv_heads self.dim = cfg.width // cfg.heads self.q = nn.Linear(cfg.width, cfg.width, bias=False) self.k = nn.Linear(cfg.width, cfg.kv_heads * self.dim, bias=False) self.v = nn.Linear(cfg.width, cfg.kv_heads * self.dim, bias=False) self.out = nn.Linear(cfg.width, cfg.width, bias=False) inv = cfg.rope_theta ** (-torch.arange(0, self.dim, 2).float() / self.dim) angle = torch.outer(torch.arange(cfg.context).float(), inv) self.register_buffer("cos", angle.cos(), persistent=False) self.register_buffer("sin", angle.sin(), persistent=False) self.dropout = cfg.dropout def forward(self, x, precision, pre_gain, post_gain, cache=None): b, t, d = x.shape q = self.q(x).view(b, t, self.heads, self.dim).transpose(1, 2) k = self.k(x).view(b, t, self.kv_heads, self.dim).transpose(1, 2) v = self.v(x).view(b, t, self.kv_heads, self.dim).transpose(1, 2) # Keep Q/K in their projected autocast dtype, not the FP32 residual dtype. q = F.normalize(q.float(), dim=-1).to(q.dtype) * math.sqrt(self.dim) k = F.normalize(k.float(), dim=-1).to(k.dtype) * math.sqrt(self.dim) past = 0 if cache is None or cache.key is None else cache.key.shape[2] cos, sin = self.cos[past:past+t].to(q.dtype), self.sin[past:past+t].to(q.dtype) def rotate(z): even, odd = z[..., ::2], z[..., 1::2] return torch.stack((even*cos-odd*sin, even*sin+odd*cos), -1).flatten(-2) q, k = rotate(q), rotate(k) q = q * precision.to(q.dtype).transpose(1, 2).unsqueeze(-1) if past: k, v = torch.cat((cache.key, k), dim=2), torch.cat((cache.value, v), dim=2) if cache is not None: cache.key, cache.value = k, v # Rectangular SDPA is_causal uses upper-left alignment, not the offset # needed by appended queries. Single-token decode can attend to all keys; # multi-token append needs an explicit offset-causal mask (True=allowed). mask = None if past and t > 1: mask = torch.arange(past+t, device=x.device)[None, :] <= ( past + torch.arange(t, device=x.device)[:, None]) # SDPA GQA keeps K/V parameter-efficient and uses the framework backend. a = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=not past, enable_gqa=self.heads != self.kv_heads, dropout_p=self.dropout if self.training else 0.0) a = a * pre_gain.to(a.dtype).transpose(1, 2).unsqueeze(-1) a = a.transpose(1, 2).reshape(b, t, d) # Postsynaptic FEATURE gain has its own D-wide decoder, after W_out. return self.out(a) * post_gain class Block(nn.Module): def __init__(self, cfg): super().__init__() self.n1, self.n2 = RMSNorm(cfg.width), RMSNorm(cfg.width) self.attn = Attention(cfg) self.up = nn.Linear(cfg.width, cfg.ffn_width, bias=False) self.gate = nn.Linear(cfg.width, cfg.ffn_width, bias=False) self.down = nn.Linear(cfg.ffn_width, cfg.width, bias=False) self.receptors = nn.Parameter(torch.zeros(cfg.width)) self.dropout = cfg.dropout def forward(self, x, precision, pre_gain, post_gain, ffn_gain, global_log, cache=None): a = self.attn(self.n1(x), precision, pre_gain, post_gain, cache) x = x + F.dropout(a, self.dropout, self.training) h = self.n2(x) # A global broadcast with bounded, layer-specific receptor sensitivity. modulation = torch.exp(global_log * (1 + .5*torch.tanh(self.receptors))) delta = self.down(F.silu(self.gate(h * modulation)) * self.up(h)) return x + ffn_gain * F.dropout(delta, self.dropout, self.training) class FeatureExpert(nn.Module): def __init__(self, cfg): super().__init__() self.up = nn.Linear(cfg.width, cfg.expert_width, bias=False) self.gate = nn.Linear(cfg.width, cfg.expert_width, bias=False) self.down = nn.Linear(cfg.expert_width, cfg.width, bias=False) def forward(self, h): return self.down(F.silu(self.gate(h)) * self.up(h)) class NCN(nn.Module): def __init__(self, cfg): super().__init__() d, c = cfg.width, cfg.controller_width self.cfg = cfg self.input_norm = RMSNorm(d) self.tonic = nn.GRU(d, c, batch_first=True) self.phasic = nn.Linear(d, c) self.fuse = nn.Linear(3*c, c) self.state_norm = RMSNorm(c, affine=False) self.compass = nn.ModuleList([nn.Linear(c, d) for _ in range(3)]) self.control_heads = nn.ModuleList([ControlHead(cfg) for _ in range(cfg.core_layers+2)]) self.global_context = nn.Linear(c, d, bias=False) self.global_baseline = nn.Parameter(torch.zeros(d)) self.route_view = nn.Linear(c, c, bias=False) self.compass_route = nn.Linear(3*d, c, bias=False) self.route_keys = nn.Parameter(torch.randn(cfg.experts, c)) self.route_bias = nn.Parameter(torch.zeros(cfg.experts)) self.route_temperature_raw = nn.Parameter(torch.tensor(0.0)) self.observation = nn.Linear(d, c, bias=False) self.error_encoder = nn.Linear(d, c, bias=False) self.feedback = nn.GRUCell(2*c, c) self.predictor = nn.Sequential(nn.Linear(d, c), nn.SiLU(), nn.Linear(c, d)) self.pass_embedding = nn.Parameter(torch.zeros(cfg.max_passes, c)) self.refresh_base = nn.Parameter(torch.full((d,), 2.0)) self.refresh_context = nn.Linear(c, d, bias=False) self.update_base = nn.Parameter(torch.tensor(-2.0)) self.update_context = nn.Linear(c, 1, bias=False) self.register_buffer("prediction_scale", torch.tensor(1.0)) def recovery_penalty(self, head, state): # Train ONLY the offending readout with this auxiliary gradient. Do not # erase the shared NCN context to satisfy a numerical range objective. # FP32, outside tanh: its restoring gradient remains alive at saturation. with torch.autocast(device_type=state.device.type, enabled=False): raw = F.linear(self.state_norm(state).detach().float(), head.weight.float()) return F.relu(raw.abs() - self.cfg.recovery_radius).square().mean() def initial(self, embeddings, ids, cache=None): x = self.input_norm(embeddings) tonic, final_tonic = self.tonic(x, None if cache is None else cache.tonic) if cache is not None and cache.length: delta = x - torch.cat((cache.previous_embedding, x[:, :-1]), dim=1) starts = torch.cat((cache.previous_token, ids[:, :-1]), dim=1).eq(self.cfg.eos_id) else: delta = F.pad(x[:, 1:] - x[:, :-1], (0, 0, 1, 0)) starts = F.pad(ids[:, :-1].eq(self.cfg.eos_id), (1, 0), value=True) if cache is not None: cache.tonic = final_tonic cache.previous_embedding, cache.previous_token = x[:, -1:].clone(), ids[:, -1:].clone() delta = delta.masked_fill(starts.unsqueeze(-1), 0) phasic = 2 * torch.tanh(self.phasic(delta) / 2) state = 2 * torch.tanh(self.fuse(torch.cat((tonic, phasic, tonic*phasic), -1)) / 2) predictions = [head(s) for head, s in zip(self.compass, (phasic, state, tonic))] return state, predictions def controls(self, state, layer, disabled): s = self.state_norm(state) controls, penalty = self.control_heads[layer](s, disabled) raw = self.global_context(s) contextual = log_control(raw, math.log(1.25), True) if "global" in disabled or "all_modulation" in disabled: contextual = torch.zeros_like(contextual) global_log = log_control(self.global_baseline, .5, True) + contextual return controls, global_log, penalty + raw.float().square().mean() def advance(self, previous, current, state, disabled): before = F.normalize(previous.detach().float(), dim=-1) * math.sqrt(self.cfg.width) after = F.normalize(current.detach().float(), dim=-1) * math.sqrt(self.cfg.width) change = after - before target = change / self.prediction_scale.detach().clamp_min(.05) # Prediction has no artificially tiny +/-0.1 range. Bound only the error # fed to the controller; targets never contain future tokens or labels. predicted = self.predictor(before.to(current.dtype)).float() prediction_loss = F.smooth_l1_loss(predicted, target) err = (target-predicted.detach()).clamp(-3, 3).to(current.dtype) obs = self.observation(after.to(current.dtype)) encoded = self.error_encoder(err) if "feedback" in disabled: encoded = torch.zeros_like(encoded) if "observation" in disabled: obs = torch.zeros_like(obs) features = torch.cat((torch.tanh(obs), torch.tanh(encoded)), -1) next_state = self.feedback(features.flatten(0, 1), state.flatten(0, 1)).view_as(state) return next_state, prediction_loss, change.square().mean().detach() def route(self, state, predictions, mode): s = self.state_norm(state) if mode == "shuffled": # Counterfactual only; never used by ordinary causal inference. s = s.roll(1, dims=1) compass = torch.cat([F.normalize(p.detach().float(), dim=-1).to(s.dtype) for p in predictions], -1) if mode == "shuffled": compass = compass.roll(1, dims=1) if mode == "no_compass": compass = torch.zeros_like(compass) if mode == "no_flavor": s, compass = torch.zeros_like(s), torch.zeros_like(compass) query = F.normalize(torch.tanh(self.route_view(s) + self.compass_route(compass)).float(), dim=-1) keys = F.normalize(self.route_keys.float(), dim=-1) temperature = .1 + 1.9*torch.sigmoid(self.route_temperature_raw) logits = query @ keys.T / temperature + self.route_bias weights = logits.softmax(-1) if mode == "uniform": weights = torch.ones_like(weights) / self.cfg.experts elif mode == "fixed": weights = self.route_bias.softmax(-1).expand_as(weights) elif mode not in {"normal", "shuffled", "no_compass", "no_flavor"}: raise ValueError(f"unknown route mode: {mode}") weights = self.cfg.route_floor/self.cfg.experts + (1-self.cfg.route_floor)*weights return weights class RecurrentWorkspace(nn.Module): def __init__(self, cfg: WorkspaceConfig): super().__init__() self.cfg = cfg self.embedding = nn.Embedding(cfg.vocab_size, cfg.width) self.prelude, self.coda = Block(cfg), Block(cfg) self.core = nn.ModuleList([Block(cfg) for _ in range(cfg.core_layers)]) self.ncn = NCN(cfg) self.expert_norm = RMSNorm(cfg.width) self.experts = nn.ModuleList([FeatureExpert(cfg) for _ in range(cfg.experts)]) self.refiner_norm = RMSNorm(cfg.width) self.refiner = nn.Sequential(nn.Linear(cfg.width, 2*cfg.refiner_width, bias=False)) self.refiner_down = nn.Linear(cfg.refiner_width, cfg.width, bias=False) self.output_norm = RMSNorm(cfg.width) self.gradient_checkpointing = False self.recovery_scale = 1.0 # trainer derives it from the saved global step self.register_buffer("ema_embedding", torch.empty(cfg.vocab_size, cfg.width)) self.apply(self._initialize) # Small, nonzero contextual readouts permit task gradients from step one. for head in self.ncn.control_heads: nn.init.normal_(head.contextual.weight, std=.001) for head in (self.ncn.global_context, self.ncn.refresh_context, self.ncn.update_context): nn.init.normal_(head.weight, std=.001) for block in (self.prelude, *self.core, self.coda): nn.init.normal_(block.attn.out.weight, std=.02/math.sqrt(2*(cfg.core_layers*cfg.passes+2))) nn.init.normal_(block.down.weight, std=.02/math.sqrt(2*(cfg.core_layers*cfg.passes+2))) self.ema_embedding.copy_(self.embedding.weight.detach()) if self.parameter_report()["total_trainable"] > cfg.parameter_cap: raise ValueError(f"parameter cap exceeded: {self.parameter_report()}") @staticmethod def _initialize(module): if isinstance(module, (nn.Linear, nn.Embedding)): nn.init.normal_(module.weight, std=.02) if getattr(module, "bias", None) is not None: nn.init.zeros_(module.bias) elif isinstance(module, (nn.GRU, nn.GRUCell)): for name, value in module.named_parameters(recurse=False): nn.init.normal_(value, std=.02) if "weight" in name else nn.init.zeros_(value) def parameter_report(self): groups = {} for name, value in self.named_parameters(): group = name.split(".")[0] groups[group] = groups.get(group, 0) + value.numel() return {"groups": groups, "total_trainable": sum(groups.values()), "teacher_buffer_elements": self.ema_embedding.numel(), "main_block_applications": 2 + self.cfg.core_layers*self.cfg.passes} @torch.no_grad() def update_teachers(self, prediction_mse): self.ema_embedding.lerp_(self.embedding.weight.detach(), .001) if prediction_mse is not None: target = torch.as_tensor(prediction_mse, device=self.ncn.prediction_scale.device).sqrt().clamp(.05, 3) self.ncn.prediction_scale.lerp_(target, .01) def target_log_probs(self, hidden, labels): flat_h, flat_y = hidden.flatten(0, 1), labels.flatten() chunks = [] def score(h, y): return -F.cross_entropy(F.linear(h, self.embedding.weight).float(), y, reduction="none", ignore_index=-100) for h, y in zip(flat_h.split(self.cfg.head_chunk), flat_y.split(self.cfg.head_chunk)): if self.gradient_checkpointing and self.training and torch.is_grad_enabled(): chunks.append(checkpoint(score, h, y, use_reentrant=False)) else: chunks.append(score(h, y)) return torch.cat(chunks).view_as(labels) def compass_loss(self, ids, predictions): components = [] eos_prefix = F.pad(ids.eq(self.cfg.eos_id).int().cumsum(1), (1, 0)) future = F.embedding(ids[:, 1:], self.ema_embedding).detach().float() for horizon, prediction in zip(self.cfg.compass_horizons, predictions): length = ids.shape[1] - horizon if length <= 0: continue target = F.avg_pool1d(future.transpose(1, 2), horizon, 1).transpose(1, 2) valid = (eos_prefix[:, horizon+1:horizon+1+length]-eos_prefix[:, :length]).eq(0) loss = F.smooth_l1_loss(prediction[:, :length].float(), target, reduction="none").mean(-1) components.append((loss*valid).sum()/valid.sum().clamp_min(1)) return torch.stack(components).mean() if components else predictions[0].sum()*0 def expert_proposals(self, u): proposals = torch.stack([expert(u) for expert in self.experts], dim=2) rms = proposals.float().square().mean(-1, keepdim=True) return proposals / torch.sqrt(1+rms).to(proposals.dtype) def consensus(self, u, weights): return (self.expert_proposals(u)*weights.unsqueeze(-1)).sum(2) def refine(self, candidate): a, b = self.refiner(self.refiner_norm(candidate)).chunk(2, -1) return self.refiner_down(F.silu(a)*b) @torch.no_grad() def probability_statistics(self, hidden, labels): """Chunked observational readouts; never retain a vocabulary-sized graph.""" totals = dict(token_entropy=0., top1_probability=0., target_probability=0., token_accuracy=0.) losses, count = [], 0 for h, y in zip(hidden.flatten(0,1).split(self.cfg.head_chunk), labels.flatten().split(self.cfg.head_chunk)): valid = y.ne(-100) if not valid.any(): continue y = y[valid] lp = F.linear(h[valid],self.embedding.weight).float().log_softmax(-1) p = lp.exp() target = lp.gather(-1,y[:,None]).squeeze(-1) losses.append(-target) totals["token_entropy"] += (-(p*lp).sum(-1)).sum().item() totals["top1_probability"] += p.max(-1).values.sum().item() totals["target_probability"] += target.exp().sum().item() totals["token_accuracy"] += lp.argmax(-1).eq(y).sum().item() count += y.numel() if not count: raise ValueError("probability probe requires valid targets") nll = torch.cat(losses) return {**{"probe_"+key:value/count for key,value in totals.items()}, "probe_nll_p50":nll.quantile(.5).item(),"probe_nll_p90":nll.quantile(.9).item(), "probe_nll_p99":nll.quantile(.99).item(),"probe_tokens":count} def forward(self, ids, labels=None, *, passes=None, route_mode="normal", disabled=frozenset(), probe=False, leave_out=None, auxiliary=True, use_cache=False, cache=None): if ids.ndim != 2 or ids.shape[0] < 1 or not 1 <= ids.shape[1] <= self.cfg.context: raise ValueError("supply a nonempty, unpadded [batch, time] within context") if labels is not None and labels.shape != ids.shape: raise ValueError("labels must align with input tokens") rounds = self.cfg.passes if passes is None else passes if not 1 <= rounds <= self.cfg.max_passes: raise ValueError("pass count outside configured range") disabled = frozenset(disabled) permitted = {"precision", "pre_gain", "post_gain", "ffn_gain", "global", "all_modulation", "feedback", "observation", "refresh", "update", "refiner", "experts", "shuffle_refresh", "shuffle_update"} if disabled-permitted: raise ValueError(f"unknown control ablation: {disabled-permitted}") if cache is not None and not use_cache: raise ValueError("supply use_cache=True when appending to inference state") working_cache = None if use_cache: if labels is not None or probe or route_mode != "normal" or disabled or leave_out is not None: raise ValueError("cache supports ordinary inference only, not training targets or interventions") working_cache = prepare_state(self, ids, rounds, cache) x = self.embedding(ids) state, predictions = self.ncn.initial(x, ids, working_cache) penalties, predictions_losses, change_squares, trace = [], [], [], [] recovery_refresh, recovery_update = [], [] recover = auxiliary and labels is not None and self.cfg.recovery_weight > 0 def feedback(previous, current, s): def advance(a, b, c): return self.ncn.advance(a, b, c, disabled) # Feedback targets are detached but the predictor/GRU graph is not. # Recompute it too; no teacher/state mutation occurs inside advance. if self.gradient_checkpointing and self.training and torch.is_grad_enabled(): return checkpoint(advance, previous, current, s, use_reentrant=False) return advance(previous, current, s) def run_block(block, layer, h, s, key, advance=True): attention_cache = (None if working_cache is None else working_cache.attention.setdefault(key, AttentionState())) def controlled(h_in, s_in): controls, global_log, penalty = self.ncn.controls(s_in, layer, disabled) out = block(h_in, controls["precision"], controls["pre_gain"], controls["post_gain"], controls["ffn_gain"], global_log, attention_cache) return out, penalty # Recompute the control-decoder graph too: D-wide postsynaptic gains # otherwise retain large activation tensors at every reused block. if self.gradient_checkpointing and self.training and torch.is_grad_enabled(): out, penalty = checkpoint(controlled, h, s, use_reentrant=False) else: out, penalty = controlled(h, s) penalties.append(penalty) if probe: controls, global_log, _ = self.ncn.controls(s, layer, disabled) trace.append({"kind": "control", "layer": layer, **{k: v.detach() for k, v in controls.items()}, "raw": self.ncn.control_heads[layer].contextual(self.ncn.state_norm(s)).detach(), "baseline_raw": self.ncn.control_heads[layer].baseline.detach(), "global_raw": self.ncn.global_context(self.ncn.state_norm(s)).detach(), "global_baseline_raw": self.ncn.global_baseline.detach(), "global": global_log.detach().exp()}) if advance: s, pl, square = feedback(h, out, s) predictions_losses.append(pl) change_squares.append(square) return out, s anchor, state = run_block(self.prelude, 0, x, state, "prelude") h = anchor intermediate = [] for r in range(rounds): state = state + self.ncn.pass_embedding[r] refresh_raw = None if r: refresh_raw = self.ncn.refresh_context(self.ncn.state_norm(state)) if recover: recovery_refresh.append(self.ncn.recovery_penalty(self.ncn.refresh_context, state)) if "shuffle_refresh" in disabled: refresh_raw = refresh_raw.roll(1, dims=1) # held-out counterfactual only refresh_delta = .5*torch.tanh(refresh_raw) if "refresh" in disabled or "all_modulation" in disabled: refresh_delta = torch.zeros_like(refresh_delta) retain = torch.sigmoid(self.ncn.refresh_base + refresh_delta) h = retain*h + (1-retain)*anchor before_pass = h for i, block in enumerate(self.core, 1): h, state = run_block(block, i, h, state, f"core.{r}.{i}") u = self.expert_norm(h) # Small experts share a common coordinate system. A smooth norm cap # preserves meaningful near-zero proposals instead of blowing them up # to unit norm. All experts execute even in counterfactual modes. weights = self.ncn.route(state, predictions, route_mode) if leave_out is not None: if not 0 <= leave_out < self.cfg.experts: raise ValueError("invalid leave-out expert") keep = F.one_hot(torch.tensor(leave_out, device=ids.device), self.cfg.experts) weights = weights*(1-keep) weights = weights/weights.sum(-1, keepdim=True).clamp_min(1e-9) if probe: proposals = self.expert_proposals(u) mu = (proposals*weights.unsqueeze(-1)).sum(2) elif self.gradient_checkpointing and self.training and torch.is_grad_enabled(): mu = checkpoint(self.consensus, u, weights, use_reentrant=False) else: mu = self.consensus(u, weights) if "experts" in disabled: mu = mu*0 candidate = h + mu if self.gradient_checkpointing and self.training and torch.is_grad_enabled(): refinement = checkpoint(self.refine, candidate, use_reentrant=False) else: refinement = self.refine(candidate) if "refiner" in disabled: refinement = refinement*0 raw_update = self.ncn.update_context(self.ncn.state_norm(state)) if recover: recovery_update.append(self.ncn.recovery_penalty(self.ncn.update_context, state)) if "shuffle_update" in disabled: raw_update = raw_update.roll(1, dims=1) # held-out counterfactual only if "update" in disabled or "all_modulation" in disabled: raw_update = raw_update*0 update_rate = torch.sigmoid(self.ncn.update_base + .5*torch.tanh(raw_update)) revised = h + update_rate*(mu+refinement) state, pl, square = feedback(h, revised, state) predictions_losses.append(pl) change_squares.append(square) h = revised if probe: trace.append({"kind": "pass", "pass": r+1, "weights": weights.detach(), "proposals": proposals.detach(), "state": state.detach(), "hidden": h.detach(), "update_rate": update_rate.detach(), "update_raw":raw_update.detach(), "refresh_raw":None if refresh_raw is None else refresh_raw.detach(), "refresh_rate":None if refresh_raw is None else retain.detach(), "change_rms": (h-before_pass).float().square().mean().sqrt().detach()}) if labels is not None and (probe or (auxiliary and self.cfg.intermediate_weight > 0)): lp = self.target_log_probs(self.output_norm(h), labels) intermediate.append(-lp[labels.ne(-100)].mean()) h, state = run_block(self.coda, self.cfg.core_layers+1, h, state, "coda", advance=False) hidden = self.output_norm(h) result = {"trace": trace, "intermediate_losses": intermediate, "prediction_loss":torch.stack(predictions_losses).mean(), "prediction_mse": torch.stack(change_squares).mean()} if labels is None: result["logits"] = F.linear(hidden, self.embedding.weight).float() if working_cache is not None: working_cache.length += ids.shape[1] result["cache"] = working_cache return result logp = self.target_log_probs(hidden, labels) valid = labels.ne(-100) if not valid.any(): raise ValueError("at least one target must be valid") task_loss = -logp[valid].mean() if probe: result["probability_statistics"] = self.probability_statistics(hidden,labels) zero = task_loss.new_zeros(()) aux = { "compass": self.compass_loss(ids, predictions) if auxiliary else zero, "prediction": torch.stack(predictions_losses).mean() if auxiliary else zero, "control": torch.stack(penalties).mean() if auxiliary else zero, "intermediate": torch.stack(intermediate).mean() if auxiliary and intermediate else zero, "recovery_refresh": torch.stack(recovery_refresh).mean() if recovery_refresh else zero, "recovery_update": torch.stack(recovery_update).mean() if recovery_update else zero, } loss = (task_loss + self.cfg.compass_weight*aux["compass"] + self.cfg.prediction_weight*aux["prediction"] + self.cfg.control_weight*aux["control"] + self.cfg.intermediate_weight*aux["intermediate"]) result.update(loss=loss, task_loss=task_loss, target_log_probs=logp, auxiliary=aux) if recover: # Equal family weighting: 256 refresh channels must not drown out # the scalar update controller. No penalty inside the responsive band. result["loss"] = loss + self.cfg.recovery_weight*self.recovery_scale*( aux["recovery_refresh"] + aux["recovery_update"])/2 return result