A real two-layer neural network is training on real MNIST digits in your browser right now — and this city is that computation. Every building opens up one piece of it, all the way down to single multiply-adds.
-For the guided route, follow the numbered signs ① → ㉑ — they trace +
For the guided route, follow the numbered signs ① → ㉚ — they trace the full story: data → forward pass → loss → backprop → optimizer → metrics, then the historical baselines, learning-rate tuning, other kinds of targets, and inference, where you draw digits for the trained model to read. Then cross the river south into the Frontier, where the data stops being images: training tricks, movie-taste embeddings, decision-tree forests, text pipelines and a tiny language model — all training live too. + Finally, take the ferry over the sea to the Foundations Isle: a real CNN + learning its own kernels, residual networks racing a skipless twin, an + optimizer derby, saliency beams, and the training loop rebuilt from scratch. Or ignore the numbers and wander — every door is open.
Keyboard: WASD / arrows to walk · E to enter & inspect ·
Esc to go back · Shift to run.
diff --git a/src/sim/learner.ts b/src/sim/learner.ts
new file mode 100644
index 0000000..918ac03
--- /dev/null
+++ b/src/sim/learner.ts
@@ -0,0 +1,333 @@
+// A fastai-style Learner built from scratch (the Learner Hall's machine):
+// the standard training loop with named events fired before/after every
+// stage, callbacks that subscribe to those events, and CancelFitException
+// control flow — the ch.16/ch.19 machinery, running a real model on the
+// real digits. The Loop Machine panel lights up as these events fire.
+
+import { Tensor, addRow, crossEntropy, matmul, relu } from "../engine/tensor";
+import { SGD } from "../engine/optim";
+import { DataLoader, MnistData, mulberry32 } from "../engine/data";
+import { Scenario } from "./scenarios";
+
+export const LOOP_EVENTS = [
+ "before_fit",
+ "before_epoch",
+ "before_batch",
+ "after_pred",
+ "after_loss",
+ "after_backward",
+ "after_step",
+ "after_batch",
+ "after_epoch",
+ "after_cancel_fit",
+ "after_fit",
+] as const;
+export type LoopEvent = (typeof LOOP_EVENTS)[number];
+
+/** the control-flow exception callbacks throw to stop training. `by` is the
+ * name of the callback that raised it, stamped by `MiniLearner.call` so the
+ * attribution stays correct for any callback, not just TerminateOnNaN. */
+export class CancelFitException extends Error {
+ by?: string;
+}
+
+/** a callback is just an object with methods named after events */
+export abstract class LabCallback {
+ abstract readonly name: string;
+ abstract readonly blurb: string;
+ enabled = false;
+ /** event handlers, looked up by event name — exactly getattr(cb, name) */
+ [ev: string]: unknown;
+}
+
+/** prints (well, records) loss & accuracy per epoch — the book's TrackResults */
+export class TrackResults extends LabCallback {
+ readonly name = "TrackResults";
+ readonly blurb = "accumulates loss and accuracy every batch, closes the books each epoch";
+ rows: { epoch: number; loss: number; acc: number }[] = [];
+ private losses: number[] = [];
+ private accs: number[] = [];
+
+ before_epoch(): void {
+ this.losses = [];
+ this.accs = [];
+ }
+
+ after_batch(l: MiniLearner): void {
+ this.losses.push(l.lastLossValue);
+ this.accs.push(l.lastBatchAcc);
+ }
+
+ after_epoch(l: MiniLearner): void {
+ if (this.losses.length === 0) return;
+ this.rows.push({
+ epoch: l.epochsDone,
+ loss: this.losses.reduce((a, b) => a + b, 0) / this.losses.length,
+ acc: this.accs.reduce((a, b) => a + b, 0) / this.accs.length,
+ });
+ if (this.rows.length > 30) this.rows.splice(0, 10);
+ }
+}
+
+/** the book's ch.19 OneCycle: linear warmup for the first 25%, linear
+ * anneal after — sets the lr before every batch */
+export class OneCycleCB extends LabCallback {
+ readonly name = "OneCycle";
+ readonly blurb = "warms the lr up over the first 25% of the cycle, anneals it back down after";
+ baseLr = 0.5;
+ horizon = 400; // steps per cycle
+ startStep = 0;
+ lrs: { step: number; lr: number }[] = [];
+
+ before_fit(l: MiniLearner): void {
+ this.startStep = l.stepsDone;
+ this.lrs = [];
+ }
+
+ before_batch(l: MiniLearner): void {
+ const pct = Math.min(1, (l.stepsDone - this.startStep) / this.horizon);
+ const pctStart = 0.25;
+ const divStart = 10;
+ let lr: number;
+ if (pct < pctStart) {
+ const u = pct / pctStart;
+ lr = ((1 - u) * this.baseLr) / divStart + u * this.baseLr;
+ } else {
+ const u = (pct - pctStart) / (1 - pctStart);
+ lr = (1 - u) * this.baseLr;
+ }
+ l.opt.lr = Math.max(lr, this.baseLr / 100);
+ this.lrs.push({ step: l.stepsDone, lr: l.opt.lr });
+ if (this.lrs.length > 1000) this.lrs.splice(0, 200);
+ }
+}
+
+/** stops training the moment the loss stops being a number */
+export class TerminateOnNaN extends LabCallback {
+ readonly name = "TerminateOnNaN";
+ readonly blurb = "raises CancelFitException the moment the loss becomes NaN or infinite";
+
+ after_loss(l: MiniLearner): void {
+ if (!isFinite(l.lastLossValue)) throw new CancelFitException("loss is not finite");
+ }
+}
+
+/**
+ * The Learner itself: one_batch with an event call at every stage, exactly
+ * the book's structure. It wraps a small two-layer digit net so the loop is
+ * cheap enough to watch in real time.
+ */
+export class MiniLearner {
+ w1: Tensor;
+ b1: Tensor;
+ w2: Tensor;
+ b2: Tensor;
+ opt: SGD;
+ loader: DataLoader;
+ data: MnistData;
+ cbs: LabCallback[];
+ readonly hidden = 16;
+ /** per-event firing counts, for the Loop Machine's lamps */
+ eventCounts: Record
conv 8→16 (3×3)
→ 16 features", false],
+ ["HEAD (new)", "concat 16+16 → 32
linear 32→16→2
same? / different?", true],
+ ] as const;
+ boxes.forEach(([name, desc, accent], i) => {
+ if (i > 0) flow.append(el("div", "flow-arrow", "→"));
+ const box = el("div", `flow-box${accent ? " flow-accent" : ""}`);
+ box.append(el("div", "flow-name", name));
+ const b = el("div", "flow-body");
+ b.innerHTML = desc;
+ box.append(b);
+ flow.append(box);
+ });
+ out.append(sec);
+ const btns = el("div", "controls-row");
+ btns.append(
+ button("✂️ cut the body from the Convolution Works", () => {
+ r.transferFrom(world.conv);
+ render();
+ }, "btn-play"),
+ button("🎲 random-init the encoder instead", () => {
+ r.resetEncoder();
+ render();
+ }),
+ button("↺ reset the head", () => {
+ r.resetHead();
+ render();
+ }),
+ );
+ out.append(btns);
+ out.append(
+ el(
+ "p",
+ "explain",
+ "The encoder's first two conv layers have exactly the Convolution Works' shapes (1→8 5×5, 8→16 3×3), so its trained kernels drop straight in. Train the Convolution Works first, then cut it here, and the pair-matcher starts from real edge- and stroke-detectors instead of noise — the whole point of transfer learning. fastai's splitter (which we implement as parameter groups) is what lets the body and head carry different learning rates.",
+ ),
+ );
+ };
+ body.append(out);
+ render();
+ },
+ });
+
+ registerPanel("atelier.siamese", {
+ title: "Siamese Bench — same digit or not?",
+ subtitle:
+ "Two images pass through the SAME encoder (shared weights), their 16-feature codes are concatenated, and a two-layer head answers same/different. Freeze the body to train only the head (fast, the fastai default first step), then unfreeze for a lower-lr fine-tune.",
+ render(body, world) {
+ const r = s(world);
+ // freeze / lr controls live outside the live region so they survive re-render
+ const switches = el("div", "controls-row");
+ const freezeLabel = el("label", "control-label");
+ const freezeBox = document.createElement("input");
+ freezeBox.type = "checkbox";
+ freezeBox.checked = r.frozen;
+ freezeBox.addEventListener("change", () => r.setFrozen(freezeBox.checked));
+ freezeLabel.append(freezeBox, document.createTextNode(" freeze the encoder (body)"));
+ switches.append(freezeLabel);
+ body.append(switches);
+ const [controls, cleanCtl] = trainerControls(world.siamese, { showLr: false });
+ body.append(controls);
+ const [live, cleanLive] = liveRegion(world.siamese, (root) => {
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("pair accuracy", `${(world.siamese.lastMetricValue * 100).toFixed(1)}%`, true),
+ chip("loss", fmt(r.lossHistory.at(-1)?.value ?? NaN, 4)),
+ chip("encoder", r.encoderSource, r.encoderSource !== "random init"),
+ chip("body lr", fmt(r.encLr, 3), !r.frozen),
+ );
+ root.append(chips);
+ // the pairs the model just trained on
+ const sec = section("The batch's pairs this step (✓ = same digit)");
+ const grid = el("div", "hstack wrap-row");
+ r.lastPairs.slice(0, 6).forEach((p) => {
+ const cell = el("div", "vstack");
+ const pairRow = el("div", "hstack");
+ const ca = digitCanvas(trainImg(world, p.a), 0, 2);
+ const cb = digitCanvas(trainImg(world, p.b), 0, 2);
+ pairRow.append(ca, cb);
+ cell.append(pairRow, el("div", "caption", p.same ? "✓ same" : "✗ different"));
+ grid.append(cell);
+ });
+ sec.append(grid);
+ root.append(sec);
+ const pts = r.lossHistory.map((p) => ({ x: p.step, y: p.value }));
+ const sampled = pts.length > 300 ? pts.filter((_, i) => i % Math.ceil(pts.length / 300) === 0) : pts;
+ root.append(lineChart(sampled, { width: 500, height: 130, yLabel: "pair loss" }));
+ root.append(
+ el(
+ "p",
+ "explain",
+ "Because both images share ONE encoder, the model can't just memorize positions — it must learn features that make two views of the same digit land near each other. Freezing the body makes only the head's ~600 weights move (train it hot and fast); unfreezing lets the whole thing fine-tune, usually at a lower body learning rate so the pretrained features aren't wrecked. The lr dials for each group live on the Cutting Desk.",
+ ),
+ );
+ }, 600);
+ body.append(live);
+ // per-group lr sliders (outside the live region)
+ const lrRow = el("div", "controls-row");
+ const mkLr = (label: string, get: () => number, set: (v: number) => void) => {
+ const wrap = el("label", "control-label", `${label} `);
+ const val = el("span", "mono", fmt(get(), 3));
+ const sl = document.createElement("input");
+ sl.type = "range";
+ sl.min = "-3";
+ sl.max = "0";
+ sl.step = "0.05";
+ sl.value = String(Math.log10(get()));
+ sl.addEventListener("input", () => {
+ const v = Math.pow(10, parseFloat(sl.value));
+ set(v);
+ val.textContent = fmt(v, 3);
+ });
+ wrap.append(sl, val);
+ return wrap;
+ };
+ lrRow.append(
+ mkLr("body lr", () => Math.max(r.encLr, 1e-3), (v) => (r.encLr = v)),
+ mkLr("head lr", () => r.headLr, (v) => (r.headLr = v)),
+ );
+ body.append(lrRow);
+ return () => {
+ cleanCtl();
+ cleanLive();
+ };
+ },
+ });
+
+ registerPanel("atelier.upsample", {
+ title: "Upsampling Desk — growing a map back up",
+ subtitle:
+ "Segmentation and other 'image out' tasks need to turn small deep feature maps back into full-size images. Two mechanisms on a real residual feature map: nearest-neighbor interpolation (copy each cell into a 2×2 block) and a transposed convolution (zero-interleave, then convolve).",
+ render(body, world) {
+ let dSel = 0;
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const render = () => {
+ out.innerHTML = "";
+ const img14 = digit14(world, dSel);
+ const maps = world.res.mapsFor(img14);
+ // channel-0 map of block 2 (7×7)
+ const S = 7 * 7;
+ const small = Float32Array.from(maps.b2.subarray(0, S));
+ const up = upsampleNearest(small, 7, 7); // 14×14
+ const kernel = Float32Array.from([0.25, 0.5, 0.25, 0.5, 1, 0.5, 0.25, 0.5, 0.25]); // smoothing
+ const tconv = convTranspose2x(small, 7, 7, kernel, 3);
+ const row = el("div", "hstack wrap-row");
+ const v0 = el("div", "vstack");
+ v0.append(heatmap(small, 7, 7, { cellSize: 14 }), el("div", "caption", "deep feature map (7×7)"));
+ const v1 = el("div", "vstack");
+ v1.append(heatmap(up, 14, 14, { cellSize: 8 }), el("div", "caption", "nearest-neighbor ×2 (14×14)"));
+ const v2 = el("div", "vstack");
+ v2.append(heatmap(tconv.data, tconv.ho, tconv.wo, { cellSize: 8 }), el("div", "caption", `transposed conv (${tconv.ho}×${tconv.wo})`));
+ row.append(v0, v1, v2);
+ out.append(row);
+ out.append(
+ el(
+ "p",
+ "explain",
+ "Nearest-neighbor is the blocky baseline; a transposed (‘stride-½’) convolution inserts zeros between the input pixels and convolves, so it can learn a smoother upscaling. Neither alone recovers fine detail — a 7×7 map just doesn't hold 196 pixels' worth of information. That's why the U-Net adds skip/cross connections from the matching-resolution layers of the encoder, feeding the high-resolution detail back in. (The residual net here is a classifier, so this desk shows the mechanism, not a full segmenter.)",
+ ),
+ );
+ };
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ render();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ );
+ body.append(controls, out);
+ render();
+ },
+ });
+}
+
+/** a train digit as floats (the siamese pairs are drawn from the train set) */
+function trainImg(world: World, i: number): Float32Array {
+ const out = new Float32Array(784);
+ for (let j = 0; j < 784; j++) out[j] = world.data.trainImages[i * 784 + j] / 255;
+ return out;
+}
diff --git a/src/ui/panels/beacon.ts b/src/ui/panels/beacon.ts
new file mode 100644
index 0000000..3ac4cb1
--- /dev/null
+++ b/src/ui/panels/beacon.ts
@@ -0,0 +1,80 @@
+// The Horizon Beacon: the tour's final stop. A look back over the whole
+// city — every district and the concept it taught — and a look out at the
+// open water: where the map ends and the reader's own projects begin.
+// (The book's concluding chapter: keep the momentum going.)
+
+import { registerPanel } from "../panel";
+import { chip, el, section } from "../widgets";
+
+interface Leg {
+ region: string;
+ idea: string;
+}
+
+const JOURNEY: Leg[] = [
+ { region: "Old town — the image pipeline", idea: "pixels as tensors, the forward pass, softmax + cross-entropy, backprop, SGD, and the metrics that watch it all" },
+ { region: "First Steps → Deployment", idea: "the pixel-similarity baseline, the 785-parameter learner, the LR finder, multi-label & regression heads, and inference on your own handwriting" },
+ { region: "The Frontier — beyond images", idea: "training tricks, collaborative-filtering embeddings, gradient-free decision forests, the text pipeline, and a language model from scratch" },
+ { region: "Kernel Shore & the Convolution Works", idea: "convolutions as shared-weight matmuls, a real CNN learning its own kernels, feature maps, receptive fields, and the colorful-dimension view of training stability" },
+ { region: "Residual Ridge & Saliency Point", idea: "skip connections that start as identities, adaptive pooling for any input size, and class-activation maps that show where the model looked" },
+ { region: "Momentum Row & the Bedrock Quarry", idea: "SGD → momentum → RMSProp → Adam, and the from-scratch foundations: matmul, broadcasting, weight init, and backprop by hand" },
+ { region: "Assembly Row & Loop Landing", idea: "cutting a body and sewing on a head (siamese transfer), and the training loop itself rebuilt as events + callbacks" },
+];
+
+export function registerBeaconPanels(): void {
+ registerPanel("beacon.journey", {
+ title: "The Horizon Beacon — the whole journey, and what's past the water",
+ subtitle:
+ "You've walked the entire city, from a single pixel to a training loop you could rebuild from scratch. Here's the map of what each stretch taught — and then the open sea, which is the part only you can chart.",
+ render(body, world) {
+ const chips = el("div", "chips-row");
+ let running = 0;
+ for (const t of world.trainers()) if (t.running) running++;
+ let steps = 0;
+ for (const t of world.trainers()) steps += t.scenario.step;
+ chips.append(
+ chip("live models in this city", String(world.trainers().length), true),
+ chip("models training right now", String(running)),
+ chip("total training steps taken", steps.toLocaleString()),
+ );
+ body.append(chips);
+
+ const sec = section("Everything the city taught you");
+ const list = el("div", "vstack");
+ JOURNEY.forEach((leg, i) => {
+ const row = el("div", "flow-box");
+ row.append(el("div", "flow-name", `${i + 1}. ${leg.region}`));
+ const b = el("div", "flow-body");
+ b.textContent = leg.idea;
+ row.append(b);
+ list.append(row);
+ });
+ sec.append(list);
+ body.append(sec);
+
+ const out = section("The open water");
+ out.append(
+ el(
+ "p",
+ "explain",
+ "Every number you inspected in this city was real — computed live by a from-scratch autograd engine, never canned. That means you've now seen, end to end and down to the arithmetic, what a deep learning model actually does when it trains. The book's own closing advice fits the view from this beacon: the most important thing now is to keep the momentum going — and, as you learned at the Optimizer Institute, momentum builds on itself.",
+ ),
+ );
+ out.append(
+ el(
+ "p",
+ "explain",
+ "Past this shoreline the map stops, because the next stretch is yours: pick a dataset you care about, get a model to overfit it, then reduce the overfitting in the order the Architecture Atelier laid out — more data, then augmentation, then a more generalizable architecture, then regularization, and only last a smaller model. Write about what you find. Ship something small and polished. Then start someone else on their own walk through the black box.",
+ ),
+ );
+ const send = el("div", "controls-row");
+ const back = el("button", "btn btn-play", "▶ set the whole city training");
+ back.addEventListener("click", () => {
+ for (const t of world.trainers()) t.setRunning(true);
+ });
+ send.append(back);
+ out.append(send);
+ body.append(out);
+ },
+ });
+}
diff --git a/src/ui/panels/common.ts b/src/ui/panels/common.ts
index 15c217c..2ba6e53 100644
--- a/src/ui/panels/common.ts
+++ b/src/ui/panels/common.ts
@@ -2,8 +2,24 @@
// and the reusable training-control bench.
import { Trainer } from "../../sim/trainer";
+import { World } from "../../sim/world";
import { button, chip, el, fmt, lineChart, slider } from "../widgets";
+/** a test digit downsampled to 14×14 (2×2 average pool), as 0..1 floats —
+ * the input size the Foundations Isle's residual net trains on. Shared by
+ * the Saliency Lighthouse and the Architecture Atelier. */
+export function digit14(world: World, i: number): Float32Array {
+ const out = new Float32Array(196);
+ const imgs = world.data.testImages;
+ const off = i * 784;
+ for (let r = 0; r < 14; r++)
+ for (let c = 0; c < 14; c++) {
+ const o = off + r * 2 * 28 + c * 2;
+ out[r * 14 + c] = (imgs[o] + imgs[o + 1] + imgs[o + 28] + imgs[o + 29]) / 4 / 255;
+ }
+ return out;
+}
+
/**
* Re-runs `render` into a container on every trainer step (throttled).
* Returns [root, cleanup]. Interactive state should live in the caller's
diff --git a/src/ui/panels/convworks.ts b/src/ui/panels/convworks.ts
new file mode 100644
index 0000000..5dd5c05
--- /dev/null
+++ b/src/ui/panels/convworks.ts
@@ -0,0 +1,305 @@
+// Convolution Works: a real CNN training live on the digits — the book's
+// simple_cnn. Watch it learn its own kernels, walk its feature maps, trace
+// a receptive field, and study training stability with the 'colorful
+// dimension' plot and a batchnorm switch.
+
+import { registerPanel } from "../panel";
+import { button, chip, digitCanvas, el, fmt, heatmap, lineChart, section } from "../widgets";
+import { liveRegion, picker, trainerControls } from "./common";
+import { World } from "../../sim/world";
+import { ConvNet, ACT_BINS, ACT_RANGE } from "../../sim/scenarios3";
+import { imageAsFloats } from "../../engine/data";
+
+function s(world: World): ConvNet {
+ return world.conv;
+}
+
+export function registerConvworksPanels(): void {
+ registerPanel("convworks.bench", {
+ title: "Training Bench — five convolutions, one prediction",
+ subtitle:
+ "The whole architecture: stride-2 convolutions halve the grid (28 → 14 → 7 → 4 → 2 → 1) while the channels double, ending in exactly 10 numbers — the logits. No flattening of 784 pixels, no giant weight matrix.",
+ render(body, world) {
+ const c = s(world);
+ const [controls, cleanCtl] = trainerControls(world.convnet);
+ body.append(controls);
+ const sec = section("learn.summary() — the real layer table");
+ const t = el("table", "num-table mono");
+ t.append(
+ el(
+ "tr",
+ "",
+ "layer kernel stride channels output params mult-adds / image ",
+ ),
+ );
+ let totalP = 0;
+ let totalM = 0;
+ c.layers.forEach((l, i) => {
+ const sp = l.spec;
+ const o = c.outSize(i);
+ const p = l.nParams(c.useBN && sp.act);
+ const m = l.macs(c.sizes[i], c.sizes[i]);
+ totalP += p;
+ totalM += m;
+ const row = el("tr");
+ row.append(
+ el("td", "", `conv${i + 1}${sp.act ? " + ReLU" + (c.useBN ? " + BN" : "") : ""}`),
+ el("td", "", `${sp.k}×${sp.k}`),
+ el("td", "", String(sp.stride)),
+ el("td", "", `${sp.cin} → ${sp.cout}`),
+ el("td", "", `${sp.cout} @ ${o}×${o}`),
+ el("td", "", String(p)),
+ el("td", "", m.toLocaleString()),
+ );
+ t.append(row);
+ });
+ sec.append(t);
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("total parameters", totalP.toLocaleString(), true),
+ chip("total mult-adds per image", totalM.toLocaleString()),
+ chip("main MLP's parameters, for comparison", (784 * 30 + 30 + 30 * 10 + 10).toLocaleString()),
+ );
+ sec.append(chips);
+ sec.append(
+ el(
+ "p",
+ "explain",
+ "Two design rules from the book: the FIRST kernel is 5×5 — computing 8 features from 3×3=9 pixels would barely compress anything, but 8 from 25 pixels forces real feature-finding. And every time a stride-2 layer halves the grid (÷4 activations), the channel count doubles, so the amount of computation per layer stays roughly constant. The final layer has no ReLU: logits must be free to go negative.",
+ ),
+ );
+ body.append(sec);
+ return cleanCtl;
+ },
+ });
+
+ registerPanel("convworks.kernels", {
+ title: "Learned Kernel Gallery — nobody drew these",
+ subtitle:
+ "The first layer's eight 5×5 kernels, live from the training run. At init they're noise; SGD shapes them into edge- and blob-detectors on its own — the Zeiler & Fergus observation, at digit scale. Compare them with the Kernel Workshop's hand-made edges next door.",
+ render(body, world) {
+ const c = s(world);
+ let dSel = 0;
+ let kSel = 0;
+ const controls = el("div", "controls-row");
+ const [live, cleanLive, refresh] = liveRegion(world.convnet, (root) => {
+ const row = el("div", "hstack wrap-row");
+ for (let f = 0; f < 8; f++) {
+ const v = el("div", "vstack");
+ const kimg = c.layers[0].w.data.subarray(f * 25, (f + 1) * 25);
+ const hm = heatmap(Float32Array.from(kimg), 5, 5, { symmetric: true, cellSize: 12 });
+ if (f === kSel) hm.style.outline = "2px solid #ffd34d";
+ hm.style.cursor = "pointer";
+ hm.addEventListener("click", () => {
+ kSel = f;
+ refresh();
+ });
+ v.append(hm, el("div", "caption", `kernel ${f + 1}`));
+ row.append(v);
+ }
+ root.append(row);
+ // the selected kernel applied to a real digit — its feature map
+ const img = imageAsFloats(world.data.testImages, dSel);
+ const { maps } = c.inferMaps(img);
+ const m0 = maps[0];
+ const fmap = Float32Array.from(m0.data.subarray(kSel * m0.size * m0.size, (kSel + 1) * m0.size * m0.size));
+ const row2 = el("div", "hstack wrap-row");
+ const v1 = el("div", "vstack");
+ v1.append(digitCanvas(img, 0, 4), el("div", "caption", "input digit"));
+ const v2 = el("div", "vstack");
+ v2.append(
+ heatmap(fmap, m0.size, m0.size, { cellSize: 8 }),
+ el("div", "caption", `kernel ${kSel + 1}'s feature map (after ReLU${c.useBN ? "+BN" : ""}), ${m0.size}×${m0.size}`),
+ );
+ row2.append(v1, v2);
+ root.append(row2);
+ root.append(
+ el(
+ "p",
+ "explain",
+ "Click a kernel to route the digit through it. Early in training the maps are mush; as the loss falls, each kernel specializes — some fire on strokes of one orientation, some on stroke endings. These 200 weights (8×5×5) replace the hand-crafted feature engineering of pre-deep-learning vision.",
+ ),
+ );
+ }, 700);
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ refresh();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ );
+ body.append(controls, live);
+ return cleanLive;
+ },
+ });
+
+ registerPanel("convworks.maps", {
+ title: "Feature Map Floor — NCHW, walked layer by layer",
+ subtitle:
+ "One digit flowing through the stack: each layer's activations as a grid of little maps (first 8 channels shown). The maps shrink (stride 2) while their meaning grows. Click any cell of the second layer's map to trace its receptive field back onto the input.",
+ render(body, world) {
+ const c = s(world);
+ let dSel = 0;
+ let rf: { r: number; c: number } | null = null;
+ const controls = el("div", "controls-row");
+ const [live, cleanLive, refresh] = liveRegion(world.convnet, (root) => {
+ const img = imageAsFloats(world.data.testImages, dSel);
+ const res = c.inferMaps(img);
+ // the input, with the traced receptive field if any
+ const inWrap = el("div", "hstack wrap-row");
+ const v0 = el("div", "vstack");
+ const dc = digitCanvas(img, 0, 4);
+ if (rf) {
+ const ctx = dc.getContext("2d")!;
+ // one cell of layer 2 (7×7) sees a 9×9 input area at 4-pixel jumps
+ const cx = rf.c * 4 - 4;
+ const cy = rf.r * 4 - 4;
+ ctx.strokeStyle = "#ffd34d";
+ ctx.lineWidth = 2;
+ ctx.strokeRect(cx * 4, cy * 4, 9 * 4, 9 * 4);
+ }
+ v0.append(dc, el("div", "caption", rf ? "receptive field of the clicked cell (9×9 pixels)" : "input (1@28×28)"));
+ inWrap.append(v0);
+ root.append(inWrap);
+ res.maps.forEach((m, li) => {
+ const sec = el("div", "vstack");
+ sec.append(el("div", "caption", `layer ${li + 1}: ${m.c}@${m.size}×${m.size} ${li === 1 ? "— click a cell to trace" : ""}`));
+ const row = el("div", "hstack wrap-row");
+ const S = m.size * m.size;
+ for (let ch = 0; ch < Math.min(m.c, 8); ch++) {
+ const map = Float32Array.from(m.data.subarray(ch * S, (ch + 1) * S));
+ const cellSize = Math.max(3, Math.min(8, Math.floor(56 / m.size)));
+ const hm = heatmap(map, m.size, m.size, {
+ cellSize,
+ onClickCell:
+ li === 1
+ ? (r2, c2) => {
+ rf = { r: r2, c: c2 };
+ refresh();
+ }
+ : undefined,
+ });
+ row.append(hm);
+ }
+ sec.append(row);
+ root.append(sec);
+ });
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("prediction", String(res.pred), res.pred === world.data.testLabels[dSel]),
+ chip("truth", String(world.data.testLabels[dSel])),
+ chip("p(prediction)", fmt(res.probs[res.pred], 3)),
+ );
+ root.append(chips);
+ root.append(
+ el(
+ "p",
+ "explain",
+ "Tensors here are batch × channels × height × width (NCHW). The deeper the layer, the more stride-2 halvings behind it — so each activation summarizes a larger patch of the input: its receptive field. That's why deeper layers get more channels: richer meaning needs more features.",
+ ),
+ );
+ }, 700);
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ rf = null;
+ refresh();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ );
+ body.append(controls, live);
+ return cleanLive;
+ },
+ });
+
+ registerPanel("convworks.stability", {
+ title: "Stability Observatory — the colorful dimension",
+ subtitle:
+ "Each vertical slice is a histogram of the penultimate layer's activations at one training step (log-brightness). Healthy training is a smooth band; 'bad training' is the crash-and-restart pattern — near-zero activations flooding the bottom. Try a hot learning rate with and without batchnorm.",
+ render(body, world) {
+ const c = s(world);
+ const controls = el("div", "controls-row");
+ const bnLabel = el("label", "control-label");
+ const bnBox = document.createElement("input");
+ bnBox.type = "checkbox";
+ bnBox.checked = c.useBN;
+ bnBox.addEventListener("change", () => (c.useBN = bnBox.checked));
+ bnLabel.append(bnBox, document.createTextNode(" batchnorm after every ReLU"));
+ controls.append(
+ bnLabel,
+ button("🌶 lr = 0.5 (hot)", () => world.convnet.setLr(0.5)),
+ button("📗 lr = 0.06 (the book's demo rate)", () => world.convnet.setLr(0.06)),
+ button("↺ restart from scratch", () => (world.conv.reinit())),
+ );
+ body.append(controls);
+ const [live, cleanLive] = liveRegion(world.convnet, (root) => {
+ const hists = c.actHists;
+ if (hists.length < 3) {
+ root.append(el("p", "explain", "train a few steps first — the observatory records one histogram per step"));
+ return;
+ }
+ // the color_dim canvas: x = step, y = activation value bin, color = log count
+ const W = 460;
+ const H = 180;
+ const canvas = document.createElement("canvas");
+ canvas.width = W;
+ canvas.height = H;
+ canvas.className = "heatmap";
+ const ctx = canvas.getContext("2d")!;
+ ctx.fillStyle = "#10131c";
+ ctx.fillRect(0, 0, W, H);
+ const n = hists.length;
+ const colW = Math.max(1, Math.floor(W / n));
+ let maxLog = 1e-9;
+ for (const h of hists) for (let b = 0; b < ACT_BINS; b++) maxLog = Math.max(maxLog, Math.log1p(h.bins[b]));
+ hists.forEach((h, i) => {
+ const x = Math.floor((i / n) * W);
+ for (let b = 0; b < ACT_BINS; b++) {
+ const u = Math.log1p(h.bins[b]) / maxLog;
+ if (u <= 0.01) continue;
+ // dark blue → teal → yellow ramp
+ const r2 = Math.floor(20 + 235 * u * u);
+ const g2 = Math.floor(30 + 200 * u);
+ const b2 = Math.floor(70 + 120 * (1 - u));
+ ctx.fillStyle = `rgb(${r2},${g2},${b2})`;
+ const y = H - Math.floor(((b + 1) / ACT_BINS) * H);
+ ctx.fillRect(x, y, colW, Math.ceil(H / ACT_BINS));
+ }
+ });
+ root.append(canvas);
+ root.append(
+ el(
+ "div",
+ "caption",
+ `activation histograms of conv4's output, last ${n} steps · y spans −${ACT_RANGE}…+${ACT_RANGE} · zero sits mid-height`,
+ ),
+ );
+ const last = c.actStats[c.actStats.length - 1];
+ if (!last) return; // actStats/actHists are pushed in lockstep, but guard anyway
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("activations near zero", `${(last.nearZero * 100).toFixed(0)}%`, last.nearZero > 0.5),
+ chip("mean", fmt(last.mean, 3)),
+ chip("std", fmt(last.std, 3)),
+ chip("batchnorm", c.useBN ? "ON" : "off", c.useBN),
+ );
+ root.append(chips);
+ const pts = c.actStats.map((a) => ({ x: a.step, y: a.nearZero }));
+ root.append(el("div", "caption", "fraction of near-zero activations over time:"));
+ root.append(lineChart(pts, { width: 460, height: 110, yLabel: "near-zero fraction" }));
+ root.append(
+ el(
+ "p",
+ "explain",
+ "Near-zero activations are dead computation — multiplying by ~0 contributes nothing, and the deadness compounds layer to layer. Batchnorm normalizes each channel with the batch's own statistics, then re-scales with two LEARNED parameters per channel (gamma·ŷ + beta), so activations stay in a healthy range at any learning rate. During training it uses batch statistics; at evaluation it switches to running averages — the same numbers this building's test-accuracy metric uses.",
+ ),
+ );
+ }, 500);
+ body.append(live);
+ const [ctl, cleanCtl] = trainerControls(world.convnet);
+ body.append(ctl);
+ return () => {
+ cleanLive();
+ cleanCtl();
+ };
+ },
+ });
+}
diff --git a/src/ui/panels/index.ts b/src/ui/panels/index.ts
index ff31a04..b10e82d 100644
--- a/src/ui/panels/index.ts
+++ b/src/ui/panels/index.ts
@@ -14,6 +14,15 @@ import { registerCollabPanels } from "./collab";
import { registerArborPanels } from "./arbor";
import { registerLanguagePanels } from "./language";
import { registerEchoPanels } from "./echo";
+import { registerKernelPanels } from "./kernels";
+import { registerConvworksPanels } from "./convworks";
+import { registerResidualPanels } from "./residual";
+import { registerSaliencyPanels } from "./saliency";
+import { registerInstitutePanels } from "./institute";
+import { registerQuarryPanels } from "./quarry";
+import { registerAtelierPanels } from "./atelier";
+import { registerLearnerHallPanels } from "./learnerhall";
+import { registerBeaconPanels } from "./beacon";
export function registerAllPanels(): void {
registerDataPanels();
@@ -33,4 +42,14 @@ export function registerAllPanels(): void {
registerArborPanels();
registerLanguagePanels();
registerEchoPanels();
+ // the foundations isle
+ registerKernelPanels();
+ registerConvworksPanels();
+ registerResidualPanels();
+ registerSaliencyPanels();
+ registerInstitutePanels();
+ registerQuarryPanels();
+ registerAtelierPanels();
+ registerLearnerHallPanels();
+ registerBeaconPanels();
}
diff --git a/src/ui/panels/institute.ts b/src/ui/panels/institute.ts
new file mode 100644
index 0000000..f3402d2
--- /dev/null
+++ b/src/ui/panels/institute.ts
@@ -0,0 +1,185 @@
+// Optimizer Institute: four identical networks race, one per optimizer.
+// The race floor plots them together; the Momentum Desk shows the moving
+// average smoothing the noisy gradient; the Adam Microscope opens one
+// parameter's full bookkeeping.
+
+import { registerPanel } from "../panel";
+import { chip, el, fmt, lineChart, section } from "../widgets";
+import { liveRegion, trainerControls } from "./common";
+import { World } from "../../sim/world";
+import { OptRace } from "../../sim/scenarios3";
+import { Momentum } from "../../engine/optim";
+
+function s(world: World): OptRace {
+ return world.race;
+}
+
+export function registerInstitutePanels(): void {
+ registerPanel("institute.race", {
+ title: "The Race Floor — one dial, four optimizers",
+ subtitle:
+ "Four two-layer nets, identical initial weights, fed the identical batches. The only difference is what turns their gradients into steps: plain SGD, SGD+momentum, RMSProp, Adam. One learning-rate dial feeds all four — which is exactly the point: each optimizer has a different sweet spot.",
+ render(body, world) {
+ const r = s(world);
+ const [controls, cleanCtl] = trainerControls(world.optrace);
+ body.append(controls);
+ const legend = el("div", "chips-row");
+ r.racers.forEach((rc) => {
+ const c = el("div", "chip");
+ c.append(el("span", "chip-label", "●"), el("span", "chip-value", rc.name));
+ (c.querySelector(".chip-label") as HTMLElement).style.color = rc.color;
+ legend.append(c);
+ });
+ body.append(legend);
+ const [live, cleanLive] = liveRegion(world.optrace, (root) => {
+ const chips = el("div", "chips-row");
+ r.racers.forEach((rc) => {
+ const acc = rc.accHistory.at(-1)?.value;
+ chips.append(chip(`${rc.name} acc`, acc === undefined ? "—" : `${(acc * 100).toFixed(1)}%`, rc.name === "Adam"));
+ });
+ root.append(chips);
+ const sample = (pts: { step: number; value: number }[]) => {
+ const mapped = pts.map((p) => ({ x: p.step, y: p.value }));
+ return mapped.length > 300 ? mapped.filter((_, i) => i % Math.ceil(mapped.length / 300) === 0) : mapped;
+ };
+ root.append(el("div", "caption", "training loss — all four optimizers:"));
+ root.append(
+ lineChart(sample(r.racers[0].lossHistory), {
+ width: 540,
+ height: 170,
+ color: r.racers[0].color,
+ extra: r.racers.slice(1).map((rc) => ({ points: sample(rc.lossHistory), color: rc.color })),
+ }),
+ );
+ if (r.racers[0].accHistory.length > 1) {
+ root.append(el("div", "caption", "test accuracy per epoch:"));
+ root.append(
+ lineChart(sample(r.racers[0].accHistory), {
+ width: 540,
+ height: 150,
+ color: r.racers[0].color,
+ extra: r.racers.slice(1).map((rc) => ({ points: sample(rc.accHistory), color: rc.color })),
+ }),
+ );
+ }
+ root.append(
+ el(
+ "p",
+ "explain",
+ "At a small learning rate plain SGD crawls while the adaptive methods (RMSProp, Adam) fly; crank the dial up and the adaptive methods may thrash while momentum stays smooth. There's no universal winner — that's why fastai defaults to Adam but exposes the choice. Move the lr dial and watch the ordering change.",
+ ),
+ );
+ }, 500);
+ body.append(live);
+ return () => {
+ cleanCtl();
+ cleanLive();
+ };
+ },
+ });
+
+ registerPanel("institute.momentum", {
+ title: "Momentum Desk — the ball rolling downhill",
+ subtitle:
+ "Momentum keeps a running average of the gradient instead of using only the latest one: grad_avg = mom·grad_avg + grad, then step by −lr·grad_avg. The raw gradient (jagged) and its moving average (smooth) are plotted live for one watched weight.",
+ render(body, world) {
+ const r = s(world);
+ const [live, cleanLive] = liveRegion(world.optrace, (root) => {
+ if (r.watched.length < 3) {
+ root.append(el("p", "explain", "train a few steps — the desk records the momentum racer's gradient each step"));
+ return;
+ }
+ const raw = r.watched.map((w) => ({ x: w.step, y: w.grad }));
+ const avg = r.watched.map((w) => ({ x: w.step, y: w.avg }));
+ root.append(el("div", "caption", `watched parameter: ${r.watchedLabel}`));
+ root.append(
+ lineChart(raw, {
+ width: 540,
+ height: 180,
+ color: "rgba(120,180,255,0.7)",
+ series2: avg,
+ color2: "#ffd34d",
+ yLabel: "gradient",
+ }),
+ );
+ const last = r.watched.at(-1)!;
+ // look the momentum racer up by type, not a fixed index (institute.race
+ // matches the same pattern) so a reordering can't mislabel it
+ const mom = r.racers.find((rc) => rc.opt instanceof Momentum)!.opt as Momentum;
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("β (momentum)", fmt(mom.mom, 2)),
+ chip("latest raw gradient", fmt(last.grad, 4)),
+ chip("moving average (used for the step)", fmt(last.avg, 4), true),
+ );
+ root.append(chips);
+ root.append(
+ el(
+ "div",
+ "bigmath",
+ "grad_avg ← β·grad_avg + grad (blue = raw grad, gold = grad_avg) → w ← w − lr·grad_avg",
+ ),
+ );
+ root.append(
+ el(
+ "p",
+ "explain",
+ "The gold line is the blue line with its noise ironed out — a heavier ball (bigger β) skips over little bumps and rolls smoothly down a narrow valley instead of bouncing wall to wall. Too big a β and it ignores real changes in the slope and overshoots. fastai's 1cycle cycles β from 0.95 down to 0.85 and back as the learning rate rises and falls.",
+ ),
+ );
+ }, 400);
+ body.append(live);
+ const [ctl, cleanCtl] = trainerControls(world.optrace);
+ body.append(ctl);
+ return () => {
+ cleanLive();
+ cleanCtl();
+ };
+ },
+ });
+
+ registerPanel("institute.adam", {
+ title: "Adam Microscope — momentum + RMSProp + unbiasing",
+ subtitle:
+ "Adam tracks TWO moving averages of one weight: the gradient (for direction, like momentum) and the gradient SQUARED (for a per-weight learning rate, like RMSProp). It also 'unbiases' the averages so they aren't dragged toward zero at the start. Here is the full bookkeeping for one watched weight, live.",
+ render(body, world) {
+ const r = s(world);
+ const [live, cleanLive] = liveRegion(world.optrace, (root) => {
+ const st = r.adamState(3); // b2[3]
+ const sec = section("The Adam update, this step");
+ const t = el("table", "num-table mono");
+ const rowsD: [string, string][] = [
+ ["weight w (b2[3])", fmt(st.weight, 5)],
+ ["this step's gradient g", fmt(st.grad, 5)],
+ ["avg ← β₁·avg + (1−β₁)·g", fmt(st.avg, 5)],
+ ["unbiased avg = avg / (1 − β₁ⁱ⁺¹)", fmt(st.unbiased, 5)],
+ ["sqr_avg ← β₂·sqr_avg + (1−β₂)·g²", fmt(st.sqrAvg, 6)],
+ ["√(sqr_avg + eps)", fmt(Math.sqrt(st.sqrAvg + 1e-5), 5)],
+ ["effective step −lr·unbiased/√(sqr_avg+eps)", fmt((-r.opt.lr * st.unbiased) / Math.sqrt(st.sqrAvg + 1e-5), 6)],
+ ["Adam's step counter i", String(st.step)],
+ ];
+ for (const [k, v] of rowsD) {
+ const row = el("tr");
+ row.append(el("td", "", k), el("td", "", v));
+ t.append(row);
+ }
+ sec.append(t);
+ root.append(sec);
+ root.append(
+ el(
+ "p",
+ "explain",
+ "Dividing by √(sqr_avg) is the adaptive part: a weight whose gradients have been small and steady gets a BIGGER effective step (the loss is flat there, push harder); a weight whose gradients thrash gets a SMALLER one (be careful). The unbiasing divisor 1 − β₁ⁱ⁺¹ starts near zero and climbs to 1, which is why early steps aren't dragged toward zero by the freshly-initialized averages. eps also caps the maximum adaptive rate — a bigger eps makes Adam behave more like plain SGD.",
+ ),
+ );
+ }, 350);
+ body.append(live);
+ const [ctl, cleanCtl] = trainerControls(world.optrace);
+ body.append(ctl);
+ return () => {
+ cleanLive();
+ cleanCtl();
+ };
+ },
+ });
+}
diff --git a/src/ui/panels/kernels.ts b/src/ui/panels/kernels.ts
new file mode 100644
index 0000000..a474274
--- /dev/null
+++ b/src/ui/panels/kernels.ts
@@ -0,0 +1,293 @@
+// Kernel Workshop: what a convolution IS — slide a hand-made 3×3 kernel
+// over a real digit, watch edges appear, count the windows, and see that
+// the whole operation is a matmul with shared (and forced-zero) weights.
+
+import { registerPanel } from "../panel";
+import { chip, digitCanvas, el, fmt, heatmap, section } from "../widgets";
+import { picker } from "./common";
+import { convOutSize, convolveImage, im2col } from "../../engine/conv";
+import { imageAsFloats } from "../../engine/data";
+
+/** the book's four edge kernels, values verbatim */
+const KERNELS: { name: string; k: Float32Array }[] = [
+ { name: "top edge", k: Float32Array.from([-1, -1, -1, 0, 0, 0, 1, 1, 1]) },
+ { name: "left edge", k: Float32Array.from([-1, 1, 0, -1, 1, 0, -1, 1, 0]) },
+ { name: "diag 1", k: Float32Array.from([0, -1, 1, -1, 1, 0, 1, 0, 0]) },
+ { name: "diag 2", k: Float32Array.from([1, -1, 0, 0, 1, -1, 0, 0, 1]) },
+];
+
+function kernelGrid(k: Float32Array, size = 3): HTMLElement {
+ const t = el("table", "num-table mono");
+ for (let r = 0; r < size; r++) {
+ const row = el("tr");
+ for (let c = 0; c < size; c++) {
+ const cell = el("td", "", fmt(k[r * size + c], 1));
+ (cell as HTMLElement).style.color =
+ k[r * size + c] > 0 ? "#5ad1c8" : k[r * size + c] < 0 ? "#e8907a" : "";
+ row.append(cell);
+ }
+ t.append(row);
+ }
+ return t;
+}
+
+export function registerKernelPanels(): void {
+ registerPanel("kernelshop.bench", {
+ title: "Edge Kernel Bench — multiply, add, slide",
+ subtitle:
+ "A kernel is a little matrix. At every position it multiplies the 3×3 patch of pixels under it, element by element, and adds the products up. That one number is the output pixel. Click anywhere on the edge map to see the arithmetic of that exact window.",
+ render(body, world) {
+ let kSel = 0;
+ let dSel = 0;
+ let cell: { r: number; c: number } | null = { r: 12, c: 14 };
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const render = () => {
+ out.innerHTML = "";
+ const img = imageAsFloats(world.data.testImages, dSel);
+ const kernel = KERNELS[kSel].k;
+ const edge = convolveImage(img, 28, 28, kernel, 3);
+ const row = el("div", "hstack wrap-row");
+ const v1 = el("div", "vstack");
+ v1.append(digitCanvas(img, 0, 5), el("div", "caption", "the digit (28×28)"));
+ const v2 = el("div", "vstack");
+ v2.append(kernelGrid(kernel), el("div", "caption", `kernel: ${KERNELS[kSel].name}`));
+ const v3 = el("div", "vstack");
+ const hm = heatmap(edge.data, edge.ho, edge.wo, {
+ symmetric: true,
+ cellSize: 5,
+ highlight: cell,
+ onClickCell: (r, c) => {
+ cell = { r, c };
+ render();
+ },
+ });
+ v3.append(hm, el("div", "caption", `edge map (${edge.ho}×${edge.wo}) — click a cell`));
+ row.append(v1, v2, v3);
+ out.append(row);
+ if (cell) {
+ const { r, c } = cell;
+ const sec = section(
+ `The window at output (${r}, ${c})`,
+ "input pixel × kernel weight for each of the 9 positions, then the sum:",
+ );
+ const t = el("table", "num-table mono");
+ let sum = 0;
+ for (let ky = 0; ky < 3; ky++) {
+ const row2 = el("tr");
+ for (let kx = 0; kx < 3; kx++) {
+ const pix = img[(r + ky) * 28 + (c + kx)];
+ const wv = kernel[ky * 3 + kx];
+ sum += pix * wv;
+ row2.append(el("td", "", `${fmt(pix, 2)}×${fmt(wv, 0)} = ${fmt(pix * wv, 2)}`));
+ }
+ t.append(row2);
+ }
+ sec.append(t);
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("sum of the 9 products", fmt(sum, 3), true),
+ chip("edge map value", fmt(edge.data[r * edge.wo + c], 3)),
+ );
+ sec.append(chips);
+ sec.append(
+ el(
+ "p",
+ "explain",
+ "The top-edge kernel returns −a1−a2−a3+a7+a8+a9: dark above, bright below → big positive number. Flip the kernel and you detect the opposite edge. That's all 'feature detection' is at this level.",
+ ),
+ );
+ out.append(sec);
+ }
+ };
+ controls.append(
+ picker("kernel", KERNELS.length, kSel, (v) => {
+ kSel = v;
+ render();
+ }, (i) => KERNELS[i].name),
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ render();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ );
+ body.append(controls, out);
+ render();
+ },
+ });
+
+ registerPanel("kernelshop.arith", {
+ title: "Stride & Padding Yard — counting the windows",
+ subtitle:
+ "How many places can a k×k kernel visit? Pad the border with zeros to keep the corners, jump 2 pixels at a time to halve the grid. The output size follows one formula — watch it hold as the window walks.",
+ render(body) {
+ let n = 7;
+ let k = 3;
+ let stride = 1;
+ let pad = 0;
+ let pos = 0;
+ const controls = el("div", "controls-row");
+ const canvasWrap = el("div");
+ const info = el("div");
+ const CELL = 26;
+ const draw = () => {
+ info.innerHTML = "";
+ canvasWrap.innerHTML = "";
+ const outSize = convOutSize(n, k, stride, pad);
+ const total = n + 2 * pad;
+ const canvas = document.createElement("canvas");
+ canvas.width = total * CELL + 2;
+ canvas.height = total * CELL + 2;
+ const ctx = canvas.getContext("2d")!;
+ for (let y = 0; y < total; y++)
+ for (let x = 0; x < total; x++) {
+ const isPad = y < pad || x < pad || y >= n + pad || x >= n + pad;
+ ctx.fillStyle = isPad ? "rgba(90, 160, 255, 0.25)" : "rgba(255,255,255,0.10)";
+ ctx.fillRect(x * CELL + 1, y * CELL + 1, CELL - 2, CELL - 2);
+ }
+ if (outSize > 0) {
+ const nPos = outSize * outSize;
+ const p = pos % nPos;
+ const oy = Math.floor(p / outSize);
+ const ox = p % outSize;
+ ctx.fillStyle = "rgba(255, 211, 77, 0.35)";
+ ctx.strokeStyle = "#ffd34d";
+ ctx.lineWidth = 2;
+ ctx.fillRect(ox * stride * CELL + 1, oy * stride * CELL + 1, k * CELL - 2, k * CELL - 2);
+ ctx.strokeRect(ox * stride * CELL + 1, oy * stride * CELL + 1, k * CELL - 2, k * CELL - 2);
+ }
+ canvasWrap.append(canvas);
+ info.append(
+ el(
+ "div",
+ "bigmath",
+ `output size = (n + 2·pad − k) // stride + 1 = (${n} + ${2 * pad} − ${k}) // ${stride} + 1 = ${outSize}`,
+ ),
+ );
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("window positions", outSize > 0 ? `${outSize}×${outSize} = ${outSize * outSize}` : "none"),
+ chip("blue cells", "zero padding"),
+ );
+ info.append(chips);
+ info.append(
+ el(
+ "p",
+ "explain",
+ "Stride 1 + padding ks//2 keeps the size unchanged (good for adding depth). Stride 2 halves it (good for shrinking the map toward a single prediction) — and when the grid halves, CNNs double the channels so each layer keeps doing comparable work.",
+ ),
+ );
+ };
+ const mkSlider = (label: string, min: number, max: number, get: () => number, set: (v: number) => void) => {
+ const wrap = el("label", "control-label", `${label} `);
+ const val = el("span", "mono", String(get()));
+ const s = document.createElement("input");
+ s.type = "range";
+ s.min = String(min);
+ s.max = String(max);
+ s.step = "1";
+ s.value = String(get());
+ s.addEventListener("input", () => {
+ set(parseInt(s.value));
+ val.textContent = s.value;
+ draw();
+ });
+ wrap.append(s, val);
+ return wrap;
+ };
+ controls.append(
+ mkSlider("image n", 5, 9, () => n, (v) => (n = v)),
+ mkSlider("kernel k", 2, 5, () => k, (v) => (k = v)),
+ mkSlider("stride", 1, 3, () => stride, (v) => (stride = v)),
+ mkSlider("padding", 0, 2, () => pad, (v) => (pad = v)),
+ );
+ body.append(controls, canvasWrap, info);
+ draw();
+ const timer = window.setInterval(() => {
+ pos++;
+ draw();
+ }, 550);
+ return () => window.clearInterval(timer);
+ },
+ });
+
+ registerPanel("kernelshop.matmul", {
+ title: "Conv-as-Matmul Plaque — a matrix with rules",
+ subtitle:
+ "A convolution is a special matrix multiplication: unfold every 3×3 window of the image into a row, and the kernel becomes the weight column. Some weights are forced to zero (the kernel can't reach those pixels) and the rest are SHARED — every row reuses the same 9 numbers.",
+ render(body, world) {
+ let dSel = 0;
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const kernel = KERNELS[0].k;
+ const render = () => {
+ out.innerHTML = "";
+ const img = imageAsFloats(world.data.testImages, dSel);
+ // a 6×6 patch from the digit's center, where there is ink
+ const patch = new Float32Array(36);
+ for (let r = 0; r < 6; r++)
+ for (let c = 0; c < 6; c++) patch[r * 6 + c] = img[(r + 11) * 28 + (c + 11)];
+ const direct = convolveImage(patch, 6, 6, kernel, 3); // 4×4 = 16 outputs
+ const { cols, rows, colsPerRow } = im2col(patch, 6, 6, 3);
+ // conv as matmul: [16 windows × 9 pixels] @ [9 × 1 kernel]
+ const viaMatmul = new Float32Array(rows);
+ for (let r = 0; r < rows; r++) {
+ let s = 0;
+ for (let j = 0; j < colsPerRow; j++) s += cols[r * colsPerRow + j] * kernel[j];
+ viaMatmul[r] = s;
+ }
+ let maxDiff = 0;
+ for (let i = 0; i < rows; i++) maxDiff = Math.max(maxDiff, Math.abs(direct.data[i] - viaMatmul[i]));
+ // the equivalent big weight matrix [16 out-pixels × 36 in-pixels]:
+ // zeros where the window can't reach, the shared kernel values elsewhere
+ const W = new Float32Array(16 * 36);
+ for (let oy = 0; oy < 4; oy++)
+ for (let ox = 0; ox < 4; ox++)
+ for (let ky = 0; ky < 3; ky++)
+ for (let kx = 0; kx < 3; kx++)
+ W[(oy * 4 + ox) * 36 + (oy + ky) * 6 + (ox + kx)] = kernel[ky * 3 + kx];
+ const row = el("div", "hstack wrap-row");
+ const v0 = el("div", "vstack");
+ v0.append(heatmap(patch, 6, 6, { cellSize: 14 }), el("div", "caption", "6×6 patch of the real digit"));
+ const v1 = el("div", "vstack");
+ v1.append(kernelGrid(kernel), el("div", "caption", "the 3×3 kernel (9 numbers, reused)"));
+ const v2 = el("div", "vstack");
+ v2.append(
+ heatmap(W, 16, 36, { symmetric: true, cellSize: 8 }),
+ el("div", "caption", "the SAME conv as a 16×36 weight matrix — dark cells are forced zeros; each row repeats the identical 9 kernel values"),
+ );
+ row.append(v0, v1, v2);
+ out.append(row);
+ const t = el("table", "num-table mono");
+ t.append(el("tr", "", "output pixel direct convolution windows @ kernel (matmul) "));
+ for (let i = 0; i < 6; i++) {
+ const r2 = el("tr");
+ r2.append(
+ el("td", "", `#${i}`),
+ el("td", "", fmt(direct.data[i], 4)),
+ el("td", "", fmt(viaMatmul[i], 4)),
+ );
+ t.append(r2);
+ }
+ out.append(t);
+ const chips = el("div", "chips-row");
+ chips.append(chip("max |difference| over all 16 outputs", fmt(maxDiff, 8), true));
+ out.append(chips);
+ out.append(
+ el(
+ "p",
+ "explain",
+ "Two constraints on the weight matrix — forced zeros and shared values — are the whole difference between a linear layer and a convolution. They encode 'nearby pixels matter, and the same feature matters everywhere', and they slash the parameter count: 9 numbers instead of 16×36 = 576.",
+ ),
+ );
+ };
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ render();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ );
+ body.append(controls, out);
+ render();
+ },
+ });
+}
diff --git a/src/ui/panels/learnerhall.ts b/src/ui/panels/learnerhall.ts
new file mode 100644
index 0000000..1a0054b
--- /dev/null
+++ b/src/ui/panels/learnerhall.ts
@@ -0,0 +1,143 @@
+// Learner Hall: the training loop rebuilt from scratch as a machine. Each
+// stage fires a named event; callbacks hook into those events; a NaN guard
+// can halt the whole thing with CancelFitException. The Loop Machine lights
+// up as events fire; the Callback Rack lets you toggle them and break it.
+
+import { registerPanel } from "../panel";
+import { button, chip, el, fmt, lineChart, section } from "../widgets";
+import { liveRegion, trainerControls } from "./common";
+import { World } from "../../sim/world";
+import { LearnerLab } from "../../sim/learner";
+import type { MiniLearner, OneCycleCB, TrackResults } from "../../sim/learner";
+import { LOOP_EVENTS } from "../../sim/learner";
+
+function lab(world: World): LearnerLab {
+ return world.lab;
+}
+
+export function registerLearnerHallPanels(): void {
+ registerPanel("hall.loop", {
+ title: "The Loop Machine — one_batch, stage by stage",
+ subtitle:
+ "The real training loop: fetch a batch, predict, compute loss, backward, step, zero grads — and at every boundary it calls self('event'), which runs every subscribed callback. The lamps below light as each event fires on the live run.",
+ render(body, world) {
+ const l = lab(world);
+ const [controls, cleanCtl] = trainerControls(world.learnerlab, { showLr: false });
+ body.append(controls);
+ const [live, cleanLive] = liveRegion(world.learnerlab, (root) => {
+ const m = l.learner;
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("steps done", String(m.stepsDone)),
+ chip("epochs done", String(m.epochsDone)),
+ chip("batch loss", fmt(m.lastLossValue, 4), true),
+ chip("batch accuracy", `${(m.lastBatchAcc * 100).toFixed(1)}%`),
+ );
+ if (m.cancelled) chips.append(chip("⛔ HALTED by", m.cancelled.by, true));
+ root.append(chips);
+ // the event ladder — recent firings glow
+ const sec = section("The event ladder (one_batch fires these top to bottom)");
+ const ladder = el("div", "vstack");
+ const recent = new Set(m.lastTrace.slice(-6));
+ for (const ev of LOOP_EVENTS) {
+ const rowEl = el("div", "hstack");
+ const lamp = el("span", "chip-value", recent.has(ev) ? "🟢" : "⚪");
+ const name = el("span", "mono", ev);
+ if (recent.has(ev)) (name as HTMLElement).style.color = "#ffd34d";
+ const count = el("span", "caption", `${m.eventCounts[ev]}×`);
+ rowEl.append(lamp, name, document.createTextNode(" "), count);
+ ladder.append(rowEl);
+ }
+ sec.append(ladder);
+ root.append(sec);
+ root.append(
+ el(
+ "p",
+ "explain",
+ "self('after_loss') is just: for every enabled callback, if it has a method named after_loss, call it. That's the entire callback system — no magic, only getattr. Because a callback can read and write the loop's state (loss, gradients, the batch), and can raise exceptions to skip a batch or stop the fit, this handful of hook points is as flexible as copy-pasting a custom training loop, without the copy-paste.",
+ ),
+ );
+ }, 300);
+ body.append(live);
+ return () => {
+ cleanCtl();
+ cleanLive();
+ };
+ },
+ });
+
+ registerPanel("hall.callbacks", {
+ title: "The Callback Rack — plug in, break it, recover",
+ subtitle:
+ "Three real callbacks you can toggle live. TrackResults keeps the epoch tables. OneCycle schedules the learning rate. TerminateOnNaN watches the loss and raises CancelFitException the instant it goes bad — flip the lr to absurd and watch it fire.",
+ render(body, world) {
+ const l = lab(world);
+ const m: MiniLearner = l.learner;
+ // callback toggles (outside live region)
+ const rack = el("div", "vstack");
+ m.cbs.forEach((cb) => {
+ const wrap = el("label", "control-label");
+ const box = document.createElement("input");
+ box.type = "checkbox";
+ box.checked = cb.enabled;
+ box.addEventListener("change", () => (cb.enabled = box.checked));
+ wrap.append(box, document.createTextNode(` ${cb.name} — ${cb.blurb}`));
+ rack.append(wrap);
+ });
+ body.append(section("The rack"));
+ body.append(rack);
+ // the "break it" controls
+ const danger = el("div", "controls-row");
+ danger.append(
+ button("💥 set lr = 1e20 (overflow the weights)", () => {
+ m.opt.lr = 1e20;
+ }, "btn-small"),
+ button("↺ reset the learner (recover)", () => {
+ l.reset();
+ }, "btn-play"),
+ );
+ body.append(danger);
+ const [controls, cleanCtl] = trainerControls(world.learnerlab, { showLr: false });
+ body.append(controls);
+ const [live, cleanLive] = liveRegion(world.learnerlab, (root) => {
+ // OneCycle schedule trace
+ const oc = m.cbByNameepoch mean loss mean accuracy "));
+ tr.rows.slice(-8).forEach((r) => {
+ const row = el("tr");
+ row.append(el("td", "", String(r.epoch)), el("td", "", fmt(r.loss, 4)), el("td", "", `${(r.acc * 100).toFixed(1)}%`));
+ t.append(row);
+ });
+ sec.append(t);
+ root.append(sec);
+ }
+ if (m.cancelled) {
+ const warn = el("div", "verdict verdict-conf");
+ warn.textContent = `⛔ ${m.cancelled.by} raised CancelFitException: ${m.cancelled.reason}. The loop caught it, ran after_cancel_fit → after_fit, and stopped. Press “reset the learner” to recover.`;
+ root.append(warn);
+ }
+ root.append(
+ el(
+ "p",
+ "explain",
+ "A nice surprise first: because this city's engine uses a numerically stable softmax (the LogSumExp trick), cross-entropy stays bounded — around 25 — no matter how violently the weights move. The loss only becomes NaN once the weights themselves overflow float32 to infinity, which is why the button uses an absurd 1e20. Turn OFF TerminateOnNaN and hit it: the forward pass produces NaN and every remaining step is silently wasted. Turn it back ON and do it again: the callback's after_loss sees the non-finite loss and raises CancelFitException, which the loop's try/except catches to shut down cleanly. The same mechanism (events + exceptions) implements mixup, mixed precision, the LR finder, and the RNN reset — each a few lines that never touch the loop itself.",
+ ),
+ );
+ }, 350);
+ body.append(live);
+ return () => {
+ cleanCtl();
+ cleanLive();
+ };
+ },
+ });
+}
diff --git a/src/ui/panels/quarry.ts b/src/ui/panels/quarry.ts
new file mode 100644
index 0000000..501f57c
--- /dev/null
+++ b/src/ui/panels/quarry.ts
@@ -0,0 +1,299 @@
+// Foundations Quarry: a neural net from the foundations. Matrix
+// multiplication three ways (and why plain Python is hopeless),
+// broadcasting rules, why weight initialization decides whether training
+// even starts, and backpropagation derived by hand and checked against the
+// engine's autograd.
+
+import { registerPanel } from "../panel";
+import { chip, el, fmt, lineChart, section } from "../widgets";
+import { Tensor, matmul, mean, relu, addRow, sub, square } from "../../engine/tensor";
+import { mulberry32 } from "../../engine/data";
+
+export function registerQuarryPanels(): void {
+ registerPanel("quarry.matmul", {
+ title: "Matmul Bench — the operation under everything",
+ subtitle:
+ "A linear layer is a matrix multiply: each output is a dot product of an input row with a weight column. Here it's computed three ways on the same small matrices — the triple loop (what a neuron literally does), and vectorized — with real timings from your browser.",
+ render(body) {
+ const out = el("div");
+ const run = () => {
+ out.innerHTML = "";
+ const rnd = mulberry32(1);
+ const M = 32;
+ const K = 64;
+ const N = 16;
+ const a = new Float32Array(M * K);
+ const b = new Float32Array(K * N);
+ for (let i = 0; i < a.length; i++) a[i] = rnd() * 2 - 1;
+ for (let i = 0; i < b.length; i++) b[i] = rnd() * 2 - 1;
+ // 1) triple loop
+ const t0 = performance.now();
+ const REP = 40;
+ const c1 = new Float32Array(M * N);
+ for (let rep = 0; rep < REP; rep++)
+ for (let i = 0; i < M; i++)
+ for (let j = 0; j < N; j++) {
+ let s = 0;
+ for (let k = 0; k < K; k++) s += a[i * K + k] * b[k * N + j];
+ c1[i * N + j] = s;
+ }
+ const t1 = performance.now();
+ // 2) engine matmul (typed arrays, the "vectorized" path here)
+ const A = new Tensor(a, [M, K]);
+ const B = new Tensor(b, [K, N]);
+ let c2: Tensor;
+ for (let rep = 0; rep < REP; rep++) c2 = matmul(A, B);
+ const t2 = performance.now();
+ let maxDiff = 0;
+ for (let i = 0; i < M * N; i++) maxDiff = Math.max(maxDiff, Math.abs(c1[i] - c2!.data[i]));
+ const sec = section(`A[${M}×${K}] · B[${K}×${N}] — ${REP} repeats`);
+ const t = el("table", "num-table mono");
+ t.append(el("tr", "", "method what it is time "));
+ const rows: [string, string, number][] = [
+ ["triple for-loop", "sum over k of a[i,k]·b[k,j] — the neuron by hand", t1 - t0],
+ ["engine matmul", "the same, but the inner sum runs in one tight typed-array loop", t2 - t1],
+ ];
+ for (const [name, what, ms] of rows) {
+ const row = el("tr");
+ row.append(el("td", "", name), el("td", "", what), el("td", "", `${fmt(ms, 2)} ms`));
+ t.append(row);
+ }
+ sec.append(t);
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("both agree to", fmt(maxDiff, 8), true),
+ chip("dot products computed", (M * N).toLocaleString()),
+ chip("multiply-adds each", (M * N * K).toLocaleString()),
+ );
+ sec.append(chips);
+ sec.append(
+ el(
+ "p",
+ "explain",
+ "In real Python (not this compiled JS) the triple loop is ~100,000× slower than PyTorch, because Python interprets every one of these multiply-adds while PyTorch dispatches to C/BLAS. That gap is why the book preaches vectorization — elementwise arithmetic and broadcasting — to keep the inner loops out of the slow language. The value in position (i,j) is a dot product, and a layer is just a batch of them.",
+ ),
+ );
+ out.append(sec);
+ // the einsum notation reminder
+ out.append(
+ el(
+ "div",
+ "bigmath",
+ "einsum: ik,kj → ij — repeated k is summed over; that IS the matrix product. (Transpose is just ij → ji.)",
+ ),
+ );
+ };
+ const btn = el("button", "btn btn-small", "↻ run the timing again");
+ btn.addEventListener("click", run);
+ body.append(btn, out);
+ run();
+ },
+ });
+
+ registerPanel("quarry.broadcast", {
+ title: "Broadcasting Bench — stretching without copying",
+ subtitle:
+ "How a bias vector adds to a whole batch, or a scalar to a whole matrix, with no wasted memory: line the shapes up from the right; a dimension of 1 is stretched to match. This is how the addRow the Linear Mills use actually works.",
+ render(body) {
+ const out = el("div");
+ const example = (title: string, aShape: string, bShape: string, resShape: string, ok: boolean, note: string) => {
+ const sec = section(title);
+ const t = el("table", "num-table mono");
+ t.append(el("tr", "", `trailing shape `));
+ t.append(el("tr", "", `tensor A ${aShape} `));
+ t.append(el("tr", "", `tensor B ${bShape} `));
+ const rowR = el("tr");
+ rowR.append(el("td", "", "result"), el("td", "", ok ? resShape : "✗ error — incompatible"));
+ (rowR.querySelector("td:last-child") as HTMLElement).style.color = ok ? "#5ad1c8" : "#e8907a";
+ t.append(rowR);
+ sec.append(t);
+ sec.append(el("p", "explain", note));
+ out.append(sec);
+ };
+ example(
+ "bias broadcast over a batch (a row vector to a matrix)",
+ "64 × 30",
+ " 30",
+ "64 × 30",
+ true,
+ "The bias (30 numbers) is added to every one of the 64 rows. The '64' dimension is filled by stretching, not copying — the stored bias stays 30 numbers, and the missing axis is given a stride of 0 so PyTorch simply doesn't move when it steps through that dimension.",
+ );
+ example(
+ "normalize with per-column stats (a scalar-like broadcast)",
+ "64 × 784",
+ " 784",
+ "64 × 784",
+ true,
+ "Subtract a 784-length mean from every image in the batch, divide by a 784-length std — exactly Normalize(). One line, no loops.",
+ );
+ example(
+ "incompatible shapes",
+ "64 × 784",
+ " 256",
+ "—",
+ false,
+ "Rule: line shapes up from the RIGHT; each pair must be equal or one must be 1. 784 vs 256 is neither, so this is an error. To broadcast down the other axis you'd insert a size-1 dimension (unsqueeze / None-indexing) to line things up.",
+ );
+ body.append(out);
+ },
+ });
+
+ registerPanel("quarry.init", {
+ title: "Init Lab — why the first random numbers matter",
+ subtitle:
+ "Stack 50 linear layers with ReLU and watch the activation scale. Too-big weights blow up to NaN; too-small collapse to zero. The Kaiming scale √(2/n_in) is the one that keeps the standard deviation near 1 all the way down — computed here on real random tensors.",
+ render(body) {
+ let scaleMode: "tiny" | "big" | "xavier" | "kaiming" = "kaiming";
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const run = () => {
+ out.innerHTML = "";
+ const rnd = mulberry32(7);
+ const nIn = 100;
+ const scaleOf = (): number => {
+ if (scaleMode === "tiny") return 0.01;
+ if (scaleMode === "big") return 1;
+ if (scaleMode === "xavier") return Math.sqrt(1 / nIn);
+ return Math.sqrt(2 / nIn);
+ };
+ const scale = scaleOf();
+ let x = Tensor.randn([64, nIn], 1, rnd);
+ const stds: { x: number; y: number }[] = [];
+ for (let layer = 0; layer < 50; layer++) {
+ const w = Tensor.randn([nIn, nIn], scale, rnd);
+ x = relu(matmul(x, w));
+ let s = 0;
+ let m = 0;
+ for (let i = 0; i < x.size; i++) m += x.data[i];
+ m /= x.size;
+ for (let i = 0; i < x.size; i++) s += (x.data[i] - m) ** 2;
+ const std = Math.sqrt(s / x.size);
+ stds.push({ x: layer + 1, y: isFinite(std) ? std : NaN });
+ }
+ const finiteStds = stds.filter((p) => isFinite(p.y));
+ const sec = section(`50 ReLU layers, weights × ${fmt(scale, 4)}`);
+ sec.append(lineChart(finiteStds.length > 1 ? finiteStds : [{ x: 1, y: 0 }, { x: 50, y: 0 }], {
+ width: 500,
+ height: 170,
+ yLabel: "activation std",
+ }));
+ const finalStd = stds[stds.length - 1].y;
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("scale used", fmt(scale, 4)),
+ chip("std after 50 layers", isFinite(finalStd) ? fmt(finalStd, 4) : "exploded → NaN", scaleMode === "kaiming"),
+ );
+ sec.append(chips);
+ sec.append(
+ el(
+ "p",
+ "explain",
+ scaleMode === "kaiming"
+ ? "Kaiming/He init, √(2/n_in): the 2 compensates for ReLU throwing away the negative half of the activations. The std stays near 1 for all 50 layers — training can actually start. This is the init every layer in DL World uses."
+ : scaleMode === "xavier"
+ ? "Xavier/Glorot init, √(1/n_in): correct for tanh, but with ReLU (which zeroes the negatives) it undershoots and the signal slowly fades toward zero. Switch to Kaiming to see it corrected."
+ : scaleMode === "big"
+ ? "Weights ~×1 make each layer multiply the scale up; after a few dozen layers the activations overflow to NaN. There is nothing to train — the forward pass already died."
+ : "Weights ~×0.01 make each layer shrink the scale; after a few layers everything is zero, gradients vanish, and nothing learns.",
+ ),
+ );
+ out.append(sec);
+ };
+ const mk = (label: string, mode: typeof scaleMode) => {
+ const b = el("button", "btn btn-small", label);
+ b.addEventListener("click", () => {
+ scaleMode = mode;
+ run();
+ });
+ return b;
+ };
+ controls.append(
+ mk("×0.01 (too small)", "tiny"),
+ mk("×1 (too big)", "big"),
+ mk("Xavier √(1/n)", "xavier"),
+ mk("Kaiming √(2/n)", "kaiming"),
+ );
+ body.append(controls, out);
+ run();
+ },
+ });
+
+ registerPanel("quarry.manual", {
+ title: "Manual Backward Desk — the chain rule by hand",
+ subtitle:
+ "The book's from-scratch backward pass for a two-layer net (lin → ReLU → lin → MSE), each layer's gradient written out — and every hand-derived gradient checked against the engine's autograd on the same random tensors. If the numbers match, the chain rule is doing exactly what you think.",
+ render(body) {
+ const out = el("div");
+ const run = () => {
+ out.innerHTML = "";
+ const rnd = mulberry32(42);
+ const B = 8;
+ const nIn = 6;
+ const nH = 4;
+ const x = Tensor.randn([B, nIn], 1, rnd);
+ const w1 = Tensor.randn([nIn, nH], Math.sqrt(2 / nIn), rnd);
+ const b1 = Tensor.zeros([nH]);
+ const w2 = Tensor.randn([nH, 1], Math.sqrt(1 / nH), rnd);
+ const b2 = Tensor.zeros([1]);
+ const y = Tensor.randn([B, 1], 1, rnd);
+ for (const t of [w1, b1, w2, b2]) t.requiresGrad = true;
+ // forward with the engine (records the graph)
+ const l1 = addRow(matmul(x, w1), b1);
+ const a1 = relu(l1);
+ const out2 = addRow(matmul(a1, w2), b2);
+ // engine loss + backward: MSE = mean((out − y)²)
+ const diff = sub(out2, y);
+ const loss = mean(square(diff));
+ loss.backward();
+ // hand-derived gradients (the book's formulas)
+ // d loss/d out = 2·diff / B ; then lin/relu backward
+ const gOut = new Float32Array(B);
+ for (let i = 0; i < B; i++) gOut[i] = (2 * diff.data[i]) / B;
+ // w2.g = a1ᵀ · gOut ; b2.g = Σ gOut
+ const gW2 = new Float32Array(nH);
+ let gB2 = 0;
+ for (let i = 0; i < B; i++) {
+ gB2 += gOut[i];
+ for (let h = 0; h < nH; h++) gW2[h] += a1.data[i * nH + h] * gOut[i];
+ }
+ const cmp = (name: string, hand: number[], auto: Float32Array) => {
+ let md = 0;
+ for (let i = 0; i < hand.length; i++) md = Math.max(md, Math.abs(hand[i] - auto[i]));
+ const row = el("tr");
+ row.append(
+ el("td", "", name),
+ el("td", "", hand.slice(0, 3).map((v) => fmt(v, 4)).join(", ") + (hand.length > 3 ? " …" : "")),
+ el("td", "", md < 1e-4 ? `✓ match (${fmt(md, 7)})` : `✗ ${fmt(md, 5)}`),
+ );
+ (row.querySelector("td:last-child") as HTMLElement).style.color = md < 1e-4 ? "#5ad1c8" : "#e8907a";
+ return row;
+ };
+ const sec = section("Hand-derived vs autograd — same tensors, same numbers");
+ const t = el("table", "num-table mono");
+ t.append(el("tr", "", "gradient by hand (first few) vs engine "));
+ t.append(cmp("∂loss/∂b2 = Σ (2·diff/B)", [gB2], b2.grad!));
+ t.append(cmp("∂loss/∂W2 = a1ᵀ · gOut", Array.from(gW2), w2.grad!));
+ sec.append(t);
+ sec.append(
+ el(
+ "div",
+ "bigmath",
+ "dL/db₂ = Σ 2·diff/B · dL/dW₂ = a₁ᵀ·(2·diff/B) · then back through ReLU: gradient passes only where l₁ > 0",
+ ),
+ );
+ sec.append(
+ el(
+ "p",
+ "explain",
+ "The backward pass is the chain rule applied once per layer, from the loss back to the inputs — start at ∂loss/∂output, and each layer turns 'gradient w.r.t. my output' into 'gradient w.r.t. my input and my weights'. ReLU's gradient is the simplest of all: 1 where the input was positive, 0 where it was clipped. The engine that powers this whole city does exactly these steps automatically; here we did the first two by hand and confirmed they agree.",
+ ),
+ );
+ out.append(sec);
+ };
+ const btn = el("button", "btn btn-small", "↻ new random tensors");
+ btn.addEventListener("click", run);
+ body.append(btn, out);
+ run();
+ },
+ });
+}
diff --git a/src/ui/panels/residual.ts b/src/ui/panels/residual.ts
new file mode 100644
index 0000000..2c917af
--- /dev/null
+++ b/src/ui/panels/residual.ts
@@ -0,0 +1,200 @@
+// Residual Works: skip connections. Two networks of identical depth train
+// on identical batches — one's blocks compute x + f(x) and start life as
+// perfect identities, the other is the same stack without the shortcut.
+
+import { registerPanel } from "../panel";
+import { barChart, chip, el, fmt, heatmap, lineChart, numberStrip, section } from "../widgets";
+import { liveRegion, picker, trainerControls } from "./common";
+import { World } from "../../sim/world";
+import { ResRace } from "../../sim/scenarios3";
+
+function s(world: World): ResRace {
+ return world.res;
+}
+
+/** mean over channels of an [C*S] map → S values (for compact map views) */
+function channelMean(data: Float32Array, c: number, S: number): Float32Array {
+ const out = new Float32Array(S);
+ for (let ch = 0; ch < c; ch++) for (let i = 0; i < S; i++) out[i] += data[ch * S + i] / c;
+ return out;
+}
+
+export function registerResidualPanels(): void {
+ registerPanel("residual.bench", {
+ title: "Twin Race Bench — does the shortcut help?",
+ subtitle:
+ "Both networks: a 5×5 stride-2 stem, two blocks of two 3×3 convs at 7×7, average pooling, a linear head — same init where shapes allow, same batches, same learning rate. The only difference: the residual model's blocks add their input back (x + f(x)).",
+ render(body, world) {
+ const r = s(world);
+ const [controls, cleanCtl] = trainerControls(world.resnet);
+ body.append(controls);
+ const [live, cleanLive] = liveRegion(world.resnet, (root) => {
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("residual test acc", `${(r.accuracy(true) * 100).toFixed(1)}%`, true),
+ chip("plain twin test acc", `${(r.accuracy(false) * 100).toFixed(1)}%`),
+ chip("residual loss", fmt(r.lossHistory.at(-1)?.value ?? NaN, 4)),
+ chip("plain loss", fmt(r.lastPlainLoss, 4)),
+ );
+ root.append(chips);
+ const sample = (pts: { step: number; value: number }[]) => {
+ const mapped = pts.map((p) => ({ x: p.step, y: p.value }));
+ return mapped.length > 300
+ ? mapped.filter((_, i) => i % Math.ceil(mapped.length / 300) === 0)
+ : mapped;
+ };
+ root.append(el("div", "caption", "training loss — residual (teal) vs plain twin (gold):"));
+ root.append(
+ lineChart(sample(r.lossHistory), { width: 520, height: 150, series2: sample(r.plainLossHistory) }),
+ );
+ if (r.metricHistory.length > 1) {
+ root.append(el("div", "caption", "test accuracy per epoch — residual (teal) vs plain (gold):"));
+ root.append(
+ lineChart(sample(r.metricHistory), { width: 520, height: 150, series2: sample(r.plainMetricHistory) }),
+ );
+ }
+ }, 600);
+ body.append(live);
+ body.append(
+ el(
+ "p",
+ "explain",
+ "Honesty note: at this depth (6 conv layers on quarter-size digits) a plain net still trains — the book's dramatic 20-vs-56-layer degradation needs real depth. What you can see live is the residual model's head start (its blocks begin as identities, so at step 0 it's already a working 1-conv model) and the smoother path that gives it. The images are 14×14 so both nets fit in one browser frame.",
+ ),
+ );
+ return () => {
+ cleanCtl();
+ cleanLive();
+ };
+ },
+ });
+
+ registerPanel("residual.skip", {
+ title: "Skip Junction — y = x + f(x)",
+ subtitle:
+ "Live maps from the most recent training step (batch's first image, averaged over channels): the block input x, what the convs made of it f(x), and their sum. The second conv of each block starts at zero, so at birth f(x) = 0 and the block IS the identity — the book's BatchZero trick, applied to the conv since these small blocks skip the BN.",
+ render(body, world) {
+ const r = s(world);
+ const [live, cleanLive] = liveRegion(world.resnet, (root) => {
+ const S = r.grid * r.grid;
+ for (let b = 0; b < 2; b++) {
+ const xNode = r.nodes[b === 0 ? `stem out (${r.ch}@7×7)` : "block 1 out"];
+ const fNode = r.nodes[`f(x) (block ${b + 1})`];
+ const sNode = r.nodes[`x + f(x) (block ${b + 1})`];
+ const oNode = r.nodes[`block ${b + 1} out`];
+ if (!xNode || !fNode || !sNode || !oNode) continue;
+ const sec = section(`Block ${b + 1}`);
+ const row = el("div", "hstack wrap-row");
+ const views: [string, Float32Array][] = [
+ ["x (input)", channelMean(xNode.data.subarray(0, r.ch * S), r.ch, S)],
+ ["f(x) (the two convs)", channelMean(fNode.data.subarray(0, r.ch * S), r.ch, S)],
+ ["x + f(x)", channelMean(sNode.data.subarray(0, r.ch * S), r.ch, S)],
+ ["ReLU(x + f(x))", channelMean(oNode.data.subarray(0, r.ch * S), r.ch, S)],
+ ];
+ views.forEach(([label, map], i) => {
+ const v = el("div", "vstack");
+ v.append(heatmap(map, r.grid, r.grid, { cellSize: 10, symmetric: i === 1 || i === 2 }));
+ v.append(el("div", "caption", label));
+ row.append(v);
+ if (i < views.length - 1) row.append(el("div", "flow-arrow", i === 1 ? "＋" : "→"));
+ });
+ sec.append(row);
+ root.append(sec);
+ }
+ const ratios = r.residualRatios();
+ const chips = el("div", "chips-row");
+ ratios.forEach((v, i) => chips.append(chip(`block ${i + 1}: ‖f(x)‖ / ‖x‖`, fmt(v, 3), v > 0.05)));
+ root.append(chips);
+ root.append(
+ el(
+ "p",
+ "explain",
+ "Watch the ratio grow from 0 as training runs: the block isn't asked to produce the whole answer, only the RESIDUAL — the difference between what it receives and what the next stage needs. Predicting a correction is easier to optimize than predicting from scratch, which is why the skip smooths the loss landscape (Li et al. 2018) and why virtually every modern vision model keeps this trick.",
+ ),
+ );
+ }, 600);
+ body.append(live);
+ return cleanLive;
+ },
+ });
+
+ registerPanel("residual.pool", {
+ title: "Pooling Overlook — any size fits",
+ subtitle:
+ "After the blocks, adaptive average pooling collapses each 7×7 channel map to its mean: 8 numbers per image, whatever the grid size. That's why a fully convolutional net accepts images it was never sized for — only the pooled averages ever reach the head.",
+ render(body, world) {
+ const r = s(world);
+ let dSel = 0;
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const render = () => {
+ out.innerHTML = "";
+ const idx = Int32Array.from([dSel]);
+ const x14 = r.batchX14(idx, world.data.testImages);
+ const { logits } = r.forwardRes(x14, false);
+ const pooled = r.nodes[`pooled features (${r.ch} per image)`];
+ // the same digit at FULL 28×28 resolution through the same weights
+ const full = new Float32Array(784);
+ for (let j = 0; j < 784; j++) full[j] = world.data.testImages[dSel * 784 + j] / 255;
+ const big = r.runAnySize(full, 28);
+ const truth = world.data.testLabels[dSel];
+ const sec1 = section("The pooled features of the trained-size input (14×14 → 7×7 maps)");
+ if (pooled) {
+ sec1.append(numberStrip(pooled.data.slice(0, r.ch), -1, 3));
+ sec1.append(el("div", "caption", `${r.ch} channel means — this vector is ALL the head ever sees`));
+ }
+ out.append(sec1);
+ const sec2 = section("The same weights, two input sizes", "logits side by side — no retraining, no resizing:");
+ const row = el("div", "hstack wrap-row");
+ const v1 = el("div", "vstack");
+ v1.append(
+ barChart(logits.data.slice(0, 10), { labels: [..."0123456789"], width: 340, height: 130, highlight: truth }),
+ el("div", "caption", "14×14 input → 7×7 maps"),
+ );
+ const v2 = el("div", "vstack");
+ v2.append(
+ barChart(big.logits.slice(0, 10), { labels: [..."0123456789"], width: 340, height: 130, highlight: truth }),
+ el("div", "caption", `28×28 input → ${big.grid}×${big.grid} maps (pooled to the same 8 numbers)`),
+ );
+ row.append(v1, v2);
+ sec2.append(row);
+ sec2.append(
+ el(
+ "p",
+ "explain",
+ "The book's caveat applies here too: averaging patches makes sense for photos where the subject can sit anywhere — for digits (which have one orientation and position) the earlier stride-to-1×1 design is arguably more honest. This station exists to show the mechanism, and it's also exactly what makes progressive resizing possible.",
+ ),
+ );
+ out.append(sec2);
+ const sec3 = section(
+ "Where the compute lives vs where the parameters live",
+ "real counts from this model — the reason ResNets start with a plain-conv stem:",
+ );
+ const t = el("table", "num-table mono");
+ t.append(el("tr", "", "stage parameters mult-adds / image "));
+ for (const rw of r.layerCosts()) {
+ const tr2 = el("tr");
+ tr2.append(el("td", "", rw.name), el("td", "", rw.params.toLocaleString()), el("td", "", rw.macs.toLocaleString()));
+ t.append(tr2);
+ }
+ sec3.append(t);
+ sec3.append(
+ el(
+ "p",
+ "explain",
+ "Early layers run over big grids → most of the computation. Late layers have many channels → most of the parameters. Keeping the stem as cheap plain convolutions (and using 1×1 'bottleneck' convs in the very deep variants) is how full-size ResNets stay fast.",
+ ),
+ );
+ out.append(sec3);
+ };
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ render();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ );
+ body.append(controls, out);
+ render();
+ },
+ });
+}
diff --git a/src/ui/panels/saliency.ts b/src/ui/panels/saliency.ts
new file mode 100644
index 0000000..9e59088
--- /dev/null
+++ b/src/ui/panels/saliency.ts
@@ -0,0 +1,172 @@
+// Saliency Lighthouse: class activation maps. Shine a light back through
+// the residual net to see WHICH pixels made it predict a digit. Plain CAM
+// (last conv × head weights) and Grad-CAM (gradients as weights, any layer).
+
+import { registerPanel } from "../panel";
+import { barChart, button, chip, el, fmt } from "../widgets";
+import { digit14, picker } from "./common";
+import { World } from "../../sim/world";
+import { ResRace } from "../../sim/scenarios3";
+
+function s(world: World): ResRace {
+ return world.res;
+}
+
+/** draw the digit with a heat overlay (7×7 cam bilinearly stretched over 14×14) */
+function heatOverlay(img14: Float32Array, cam: Float32Array, grid: number, scale = 12): HTMLCanvasElement {
+ const canvas = document.createElement("canvas");
+ canvas.width = 14 * scale;
+ canvas.height = 14 * scale;
+ canvas.className = "digit";
+ const ctx = canvas.getContext("2d")!;
+ // base digit in grayscale
+ for (let r = 0; r < 14; r++)
+ for (let c = 0; c < 14; c++) {
+ const v = Math.max(0, Math.min(1, img14[r * 14 + c])) * 200;
+ ctx.fillStyle = `rgb(${v},${v},${v + 12})`;
+ ctx.fillRect(c * scale, r * scale, scale, scale);
+ }
+ // cam range for normalization
+ let mn = Infinity;
+ let mx = -Infinity;
+ for (const v of cam) {
+ mn = Math.min(mn, v);
+ mx = Math.max(mx, v);
+ }
+ const range = Math.max(mx - mn, 1e-9);
+ // sample the grid×grid map at each 14×14 pixel (nearest-ish), magma-ish overlay
+ for (let r = 0; r < 14; r++)
+ for (let c = 0; c < 14; c++) {
+ const gr = Math.min(grid - 1, Math.floor((r / 14) * grid));
+ const gc = Math.min(grid - 1, Math.floor((c / 14) * grid));
+ const u = (cam[gr * grid + gc] - mn) / range;
+ if (u < 0.02) continue;
+ const rr = Math.floor(40 + 215 * u);
+ const gg = Math.floor(10 + 120 * u * u);
+ const bb = Math.floor(80 + 100 * (1 - u));
+ ctx.fillStyle = `rgba(${rr},${gg},${bb},${0.55 * u + 0.1})`;
+ ctx.fillRect(c * scale, r * scale, scale, scale);
+ }
+ return canvas;
+}
+
+export function registerSaliencyPanels(): void {
+ registerPanel("lighthouse.cam", {
+ title: "The Lamp Room — Class Activation Maps",
+ subtitle:
+ "The last block's activations, weighted by the head's weight column for a class, tell you where on the 7×7 grid that class 'lit up'. Bright = the model looked here to decide. This is the einsum('ck,kij->cij') running on the live residual net.",
+ render(body, world) {
+ const r = s(world);
+ let dSel = 0;
+ let clsSel = -1; // -1 = the predicted class
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const render = () => {
+ out.innerHTML = "";
+ const img = digit14(world, dSel);
+ const truth = world.data.testLabels[dSel];
+ // predicted class first
+ const probe = r.camFor(img, 0);
+ const cls = clsSel < 0 ? probe.pred : clsSel;
+ const { cam, probs, pred } = r.camFor(img, cls);
+ const row = el("div", "hstack wrap-row");
+ const v1 = el("div", "vstack");
+ v1.append(heatOverlay(img, cam, r.grid, 12), el("div", "caption", `“looking for ${cls}” heatmap`));
+ const v2 = el("div", "vstack");
+ v2.append(
+ barChart(probs.slice(0, 10), { labels: [..."0123456789"], width: 340, height: 130, highlight: truth }),
+ el("div", "caption", "class probabilities"),
+ );
+ row.append(v1, v2);
+ out.append(row);
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("prediction", String(pred), pred === truth),
+ chip("truth", String(truth)),
+ chip("heatmap class", String(cls), cls === pred),
+ chip("p(class)", fmt(probs[cls], 3)),
+ );
+ out.append(chips);
+ out.append(
+ el(
+ "p",
+ "explain",
+ "Each of the 8 final channels has a spatial map and a single weight into the class's logit; the heatmap is their weighted sum, so it shows which regions pushed that logit up. Ask for the map of a WRONG class and it lights up wherever that digit's evidence would be — a direct window into the model's confusions. Plain CAM only works at the last conv, because there the 'weights' are literally the head's weights. For inner layers, use the Grad Beam.",
+ ),
+ );
+ };
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ render();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ picker("heatmap for class", 11, 0, (v) => {
+ clsSel = v === 0 ? -1 : v - 1;
+ render();
+ }, (i) => (i === 0 ? "predicted class" : String(i - 1))),
+ );
+ body.append(controls, out);
+ render();
+ body.append(
+ el("p", "explain", "Train the Residual Works next door and revisit — a sharper model produces a tighter, more sensible beam."),
+ );
+ },
+ });
+
+ registerPanel("lighthouse.gradcam", {
+ title: "The Grad Beam — Grad-CAM at any depth",
+ subtitle:
+ "Grad-CAM backpropagates the class score to a chosen layer's activations, averages those gradients per channel, and uses them as weights. It's a real backward pass through the recorded graph — the 'backward hook' is just reading .grad off a node. Works at any block, not only the last.",
+ render(body, world) {
+ const r = s(world);
+ let dSel = 0;
+ let blockSel = 1; // last block by default
+ const controls = el("div", "controls-row");
+ const out = el("div");
+ const render = () => {
+ out.innerHTML = "";
+ const img = digit14(world, dSel);
+ const truth = world.data.testLabels[dSel];
+ const probe = r.camFor(img, 0);
+ const cls = probe.pred;
+ const row = el("div", "hstack wrap-row");
+ for (let b = 0; b < 2; b++) {
+ const cam = r.gradCamFor(img, cls, b);
+ const v = el("div", "vstack");
+ const canvas = heatOverlay(img, cam, r.grid, 12);
+ if (b === blockSel) canvas.style.outline = "2px solid #ffd34d";
+ v.append(canvas, el("div", "caption", `block ${b + 1} Grad-CAM`));
+ row.append(v);
+ }
+ out.append(row);
+ const chips = el("div", "chips-row");
+ chips.append(
+ chip("prediction", String(cls), cls === truth),
+ chip("truth", String(truth)),
+ chip("gradient source", `logit[${cls}]`, true),
+ );
+ out.append(chips);
+ out.append(
+ el(
+ "p",
+ "explain",
+ "For a single image and a single class we CAN call score.backward() (a scalar), and the gradient of that score with respect to a block's activations tells us how much each activation mattered. Averaging the gradients over the 7×7 grid gives one weight per channel; weighting the activations by them gives the map. Earlier blocks produce coarser, more structural heat; the last block is sharpest. Every number here comes from one real backward pass — the same engine the Backprop Works uses.",
+ ),
+ );
+ };
+ controls.append(
+ picker("digit", 40, dSel, (v) => {
+ dSel = v;
+ render();
+ }, (i) => `test #${i} (a ${world.data.testLabels[i]})`),
+ picker("outline block", 2, blockSel, (v) => {
+ blockSel = v;
+ render();
+ }, (i) => `block ${i + 1}`),
+ button("↻ recompute (current weights)", render),
+ );
+ body.append(controls, out);
+ render();
+ },
+ });
+}
diff --git a/src/ui/widgets.ts b/src/ui/widgets.ts
index f160dce..a976a00 100644
--- a/src/ui/widgets.ts
+++ b/src/ui/widgets.ts
@@ -236,6 +236,8 @@ export interface LineChartOpts {
markers?: { x: number; label: string; color: string }[];
series2?: { x: number; y: number }[];
color2?: string;
+ /** any number of additional series (the Optimizer Institute's 4-way race) */
+ extra?: { points: { x: number; y: number }[]; color: string }[];
}
export function lineChart(
@@ -258,7 +260,7 @@ export function lineChart(
return canvas;
}
const tx = (x: number) => (opts.logX ? Math.log10(Math.max(x, 1e-12)) : x);
- const all = [...points, ...(opts.series2 ?? [])];
+ const all = [...points, ...(opts.series2 ?? []), ...(opts.extra ?? []).flatMap((s) => s.points)];
const xs = all.map((p) => tx(p.x));
const xmin = Math.min(...xs);
const xmax = Math.max(...xs);
@@ -302,6 +304,7 @@ export function lineChart(
};
drawSeries(points, opts.color ?? "#5ad1c8");
if (opts.series2) drawSeries(opts.series2, opts.color2 ?? "#ffd34d");
+ for (const s of opts.extra ?? []) if (s.points.length > 1) drawSeries(s.points, s.color);
for (const m of opts.markers ?? []) {
ctx.strokeStyle = m.color;
ctx.setLineDash([4, 3]);
diff --git a/src/world/buildings.ts b/src/world/buildings.ts
index 3dd0cec..b2f828a 100644
--- a/src/world/buildings.ts
+++ b/src/world/buildings.ts
@@ -29,7 +29,21 @@ export interface BuildingDef {
color: string;
roof: string;
/** which trainer powers this building's "machines are on" glow */
- trainer: "main" | "cottage" | "workshop" | "studio" | "refinery" | "cinema" | "sentiment" | "echo" | null;
+ trainer:
+ | "main"
+ | "cottage"
+ | "workshop"
+ | "studio"
+ | "refinery"
+ | "cinema"
+ | "sentiment"
+ | "echo"
+ | "convnet"
+ | "resnet"
+ | "optrace"
+ | "siamese"
+ | "learnerlab"
+ | null;
blurb: string;
interior: { w: number; h: number; stations: StationDef[] };
}
@@ -395,6 +409,138 @@ export const BUILDINGS: BuildingDef[] = [
{ id: "echo.generate", name: "The Counting Machine", icon: "📜" },
]),
},
+ // ====================================================== the foundations isle
+ // Across the sea (take the ferry): back to images, but deeper — and then
+ // under the hood of training itself. Everything here is built from
+ // scratch: convolutions, skip connections, optimizers, the Learner.
+ {
+ id: "kernelshop",
+ name: "Kernel Workshop",
+ district: "Kernel Shore",
+ tour: 22,
+ icon: "🔍",
+ x: 5, y: 79, w: 8, h: 6,
+ color: "#d9b98f", roof: "#a6825a",
+ trainer: null,
+ blurb: "A convolution is just multiply-and-add: slide a little 3×3 kernel over a digit and watch edges appear.",
+ interior: room(18, 10, [
+ { id: "kernelshop.bench", name: "Edge Kernel Bench", icon: "🔍" },
+ { id: "kernelshop.arith", name: "Stride & Padding Yard", icon: "📐" },
+ { id: "kernelshop.matmul", name: "Conv-as-Matmul Plaque", icon: "✖️" },
+ ]),
+ },
+ {
+ id: "convworks",
+ name: "Convolution Works",
+ district: "Kernel Shore",
+ tour: 23,
+ icon: "🏗",
+ x: 15, y: 79, w: 9, h: 6,
+ color: "#8fb4d9", roof: "#5a7fa6",
+ trainer: "convnet",
+ blurb: "A real CNN training live: five stride-2 convolutions, 28×28 down to one prediction — and it learns its own kernels.",
+ interior: room(20, 11, [
+ { id: "convworks.bench", name: "Training Bench", icon: "🎛" },
+ { id: "convworks.kernels", name: "Learned Kernel Gallery", icon: "🖼" },
+ { id: "convworks.maps", name: "Feature Map Floor", icon: "🗺" },
+ { id: "convworks.stability", name: "Stability Observatory", icon: "🌡" },
+ ]),
+ },
+ {
+ id: "residual",
+ name: "Residual Works",
+ district: "Residual Ridge",
+ tour: 24,
+ icon: "🌉",
+ x: 33, y: 79, w: 9, h: 6,
+ color: "#a3c9a0", roof: "#6a9666",
+ trainer: "resnet",
+ blurb: "x + f(x): blocks that start as pure identity and learn only the difference — racing a skipless twin on the same batches.",
+ interior: room(18, 10, [
+ { id: "residual.bench", name: "Twin Race Bench", icon: "🎛" },
+ { id: "residual.skip", name: "Skip Junction", icon: "🌉" },
+ { id: "residual.pool", name: "Pooling Overlook", icon: "🔭" },
+ ]),
+ },
+ {
+ id: "lighthouse",
+ name: "Saliency Lighthouse",
+ district: "Saliency Point",
+ tour: 25,
+ icon: "🗼",
+ x: 45, y: 79, w: 6, h: 6,
+ color: "#e0d08a", roof: "#b09a45",
+ trainer: "resnet",
+ blurb: "Shine a light back into the network: which pixels made it say '3'? Class activation maps, from the live model.",
+ interior: room(16, 10, [
+ { id: "lighthouse.cam", name: "The Lamp Room (CAM)", icon: "💡" },
+ { id: "lighthouse.gradcam", name: "The Grad Beam", icon: "🔦" },
+ ]),
+ },
+ {
+ id: "atelier",
+ name: "Architecture Atelier",
+ district: "Assembly Row",
+ tour: 26,
+ icon: "✂️",
+ x: 44, y: 90, w: 9, h: 6,
+ color: "#c9a3c4", roof: "#96678f",
+ trainer: "siamese",
+ blurb: "Cut a trained body, sew on a fresh head: a siamese pair-matcher built from the Convolution Works' own weights.",
+ interior: room(18, 10, [
+ { id: "atelier.cut", name: "The Cutting Desk", icon: "✂️" },
+ { id: "atelier.siamese", name: "Siamese Bench", icon: "👯" },
+ { id: "atelier.upsample", name: "Upsampling Desk", icon: "🔎" },
+ ]),
+ },
+ {
+ id: "institute",
+ name: "Optimizer Institute",
+ district: "Momentum Row",
+ tour: 27,
+ icon: "🏁",
+ x: 33, y: 90, w: 8, h: 6,
+ color: "#d98f8f", roof: "#a65a5a",
+ trainer: "optrace",
+ blurb: "Four identical networks, four optimizers, the same batches: SGD vs momentum vs RMSProp vs Adam, live.",
+ interior: room(18, 10, [
+ { id: "institute.race", name: "The Race Floor", icon: "🏁" },
+ { id: "institute.momentum", name: "Momentum Desk", icon: "🎢" },
+ { id: "institute.adam", name: "Adam Microscope", icon: "🔬" },
+ ]),
+ },
+ {
+ id: "quarry",
+ name: "Foundations Quarry",
+ district: "Bedrock Quarter",
+ tour: 28,
+ icon: "⛏",
+ x: 15, y: 90, w: 9, h: 6,
+ color: "#b9b3a4", roof: "#82796a",
+ trainer: null,
+ blurb: "Everything from scratch: matmul with three loops (and why that's 1000× too slow), broadcasting, init, backward by hand.",
+ interior: room(20, 11, [
+ { id: "quarry.matmul", name: "Matmul Bench", icon: "⛏" },
+ { id: "quarry.broadcast", name: "Broadcasting Bench", icon: "📡" },
+ { id: "quarry.init", name: "Init Lab", icon: "🧪" },
+ { id: "quarry.manual", name: "Manual Backward Desk", icon: "✍️" },
+ ]),
+ },
+ {
+ id: "hall",
+ name: "Learner Hall",
+ district: "Loop Landing",
+ tour: 29,
+ icon: "🔁",
+ x: 5, y: 90, w: 8, h: 6,
+ color: "#8fc9d9", roof: "#5a8fa6",
+ trainer: "learnerlab",
+ blurb: "The training loop as a machine: every stage fires an event, and callbacks — including a NaN guard — hook right in.",
+ interior: room(18, 10, [
+ { id: "hall.loop", name: "The Loop Machine", icon: "🔁" },
+ { id: "hall.callbacks", name: "The Callback Rack", icon: "🎚" },
+ ]),
+ },
];
export function buildingById(id: string): BuildingDef {
diff --git a/src/world/city.ts b/src/world/city.ts
index 685b670..904a2be 100644
--- a/src/world/city.ts
+++ b/src/world/city.ts
@@ -7,7 +7,7 @@ import { mulberry32 } from "../engine/data";
export const TILE = 32;
export const MAP_W = 66;
-export const MAP_H = 65;
+export const MAP_H = 106;
/**
* District lawn signs: dark placards on posts, planted on the grass strip
@@ -44,6 +44,17 @@ const DISTRICT_SIGNS: SignDef[] = [
{ text: "REFINEMENT ROW", x: 46, y: 42.5 },
{ text: "LANGUAGE LANE", x: 19, y: 53.5 },
{ text: "SEQUENCE SUMMIT", x: 47.5, y: 53.5 },
+ // the foundations isle (across the sea: back to images — then under the
+ // hood of training itself). The ferry sign marks the only way over.
+ { text: "⛴ FERRY — THE FOUNDATIONS ISLE", x: 38.5, y: 64.4 },
+ { text: "THE FOUNDATIONS — EVERYTHING FROM SCRATCH", x: 44, y: 75.4, big: true },
+ { text: "KERNEL SHORE", x: 14, y: 77.6 },
+ { text: "RESIDUAL RIDGE", x: 37.5, y: 77.6 },
+ { text: "SALIENCY POINT", x: 49, y: 77.6 },
+ { text: "LOOP LANDING", x: 9, y: 88.6 },
+ { text: "BEDROCK QUARTER", x: 19.5, y: 88.6 },
+ { text: "MOMENTUM ROW", x: 37, y: 88.6 },
+ { text: "ASSEMBLY ROW", x: 48.5, y: 88.6 },
];
/** rough tile footprint of a sign (board + posts), for decor clearing */
@@ -70,6 +81,10 @@ const enum T {
Flower = 8,
/** the tour kiosk (solid; interactable — express ride to stop ①) */
Kiosk = 9,
+ /** island shoreline (walkable) */
+ Sand = 10,
+ /** ferry pier planks (walkable; standing at the tip offers the ride) */
+ Dock = 11,
}
export class City {
@@ -125,6 +140,33 @@ export class City {
this.rect(2, 39, 2, 2, T.Road);
this.rect(30, 39, 2, 2, T.Road);
this.rect(MAP_W - 4, 39, 2, 2, T.Road);
+ // the sea between the frontier and the Foundations Isle: no bridge —
+ // the only way across is the ferry (piers at x30-31)
+ this.rect(0, 66, MAP_W, 8, T.Water); // open water
+ this.rect(0, 102, MAP_W, 4, T.Water); // south of the island
+ this.rect(0, 74, 3, 28, T.Water); // west of the island
+ this.rect(63, 74, 3, 28, T.Water); // east of the island
+ // the island's sand ring
+ this.rect(3, 74, 60, 1, T.Sand);
+ this.rect(3, 101, 60, 1, T.Sand);
+ this.rect(3, 74, 1, 28, T.Sand);
+ this.rect(62, 74, 1, 28, T.Sand);
+ // island roads: two horizontals (one per building row), verticals at
+ // the edges, and the pier landing running down the center
+ this.rect(4, 86, 58, 2, T.Road);
+ this.rect(4, 97, 58, 2, T.Road);
+ this.rect(4, 86, 2, 13, T.Road);
+ this.rect(60, 86, 2, 13, T.Road);
+ this.rect(30, 74, 2, 12, T.Road);
+ // ferry approach (mainland) + the two piers; the water between them is
+ // crossed only by boat
+ this.rect(30, 63, 2, 3, T.Road);
+ this.rect(30, 66, 2, 2, T.Dock);
+ this.rect(30, 72, 2, 2, T.Dock);
+ // the Horizon Beacon's terrace at the island's south-west point — the
+ // last stop of the tour, looking out over open water
+ this.rect(4, 99, 10, 2, T.Plaza);
+ this.rect(8, 99, 2, 2, T.Monument);
// buildings (door punched into bottom wall)
BUILDINGS.forEach((b, i) => {
this.rect(b.x, b.y, b.w, b.h, T.Building, i);
@@ -137,15 +179,13 @@ export class City {
y++;
}
});
- // border trees + scattered decor
+ // border trees (mainland only — south of y66 the map edge is open sea)
const rand = mulberry32(5150);
for (let x = 0; x < MAP_W; x++) {
this.set(x, 0, T.Tree);
this.set(x, 1, T.Tree);
- this.set(x, MAP_H - 1, T.Tree);
- if (rand() < 0.7) this.set(x, MAP_H - 2, T.Tree);
}
- for (let y = 0; y < MAP_H; y++) {
+ for (let y = 0; y < 66; y++) {
this.set(0, y, T.Tree);
this.set(1, y, T.Tree);
this.set(MAP_W - 1, y, T.Tree);
@@ -154,7 +194,7 @@ export class City {
const zones = DISTRICT_SIGNS.map(signZone);
const inSignZone = (x: number, y: number) =>
zones.some((z) => x >= z.x0 && x <= z.x1 && y >= z.y0 && y <= z.y1);
- for (let i = 0; i < 380; i++) {
+ for (let i = 0; i < 520; i++) {
const x = 2 + Math.floor(rand() * (MAP_W - 4));
const y = 2 + Math.floor(rand() * (MAP_H - 4));
if (this.get(x, y) !== T.Grass) continue;
@@ -181,6 +221,19 @@ export class City {
return this.pulses.some((p) => Math.hypot(px - p.x, py - p.y) < TILE * 1.4);
}
+ /** standing on a ferry pier? returns where the boat would take you */
+ ferryAt(tx: number, ty: number): "island" | "mainland" | null {
+ if (tx < 29 || tx > 32) return null;
+ if (ty >= 65 && ty <= 67) return "island";
+ if (ty >= 72 && ty <= 74) return "mainland";
+ return null;
+ }
+
+ /** next to the Horizon Beacon on the island's south-west terrace? */
+ beaconNear(tx: number, ty: number): boolean {
+ return tx >= 6 && tx <= 11 && ty >= 98 && ty <= 101;
+ }
+
/** building whose door is at/adjacent to this tile, for the enter prompt */
doorAt(x: number, y: number): BuildingDef | null {
for (const [dx, dy] of [[0, 0], [0, -1], [1, 0], [-1, 0], [0, 1]]) {
@@ -217,6 +270,7 @@ export class City {
this.drawTourArrows(ctx);
this.drawMonument(ctx, time, world);
+ this.drawBeacon(ctx, time);
this.drawKiosk(ctx, time);
this.drawDistrictLabels(ctx);
this.drawLoopPulses(ctx, time, world);
@@ -271,6 +325,28 @@ export class City {
ctx.fill();
break;
}
+ case T.Sand: {
+ ctx.fillStyle = checker ? "#e6d59e" : "#e0cf96";
+ ctx.fillRect(px, py, TILE, TILE);
+ if ((x * 13 + y * 7) % 6 === 0) {
+ ctx.fillStyle = "rgba(160, 130, 70, 0.35)";
+ ctx.fillRect(px + 8, py + 10, 3, 3);
+ ctx.fillRect(px + 20, py + 20, 3, 3);
+ }
+ break;
+ }
+ case T.Dock: {
+ // water underneath, then weathered planks
+ ctx.fillStyle = "#5aa7d9";
+ ctx.fillRect(px, py, TILE, TILE);
+ ctx.fillStyle = "#9a7544";
+ ctx.fillRect(px + 1, py, TILE - 2, TILE);
+ ctx.fillStyle = "rgba(0,0,0,0.22)";
+ for (let i = 0; i < 4; i++) ctx.fillRect(px + 1, py + 6 + i * 8, TILE - 2, 2);
+ ctx.fillStyle = "rgba(255,255,255,0.10)";
+ ctx.fillRect(px + 1, py, TILE - 2, 3);
+ break;
+ }
case T.Monument:
case T.Kiosk: {
// plaza floor under the structures; drawMonument/drawKiosk paint
@@ -495,6 +571,70 @@ export class City {
}
}
+ /** the Horizon Beacon: the island's (and the tour's) final stop — a small
+ * lighthouse on the south-west terrace, sweeping its light over the sea */
+ private drawBeacon(ctx: CanvasRenderingContext2D, time: number): void {
+ const cx = 9 * TILE; // center of the 2×2 monument at x8-9, y99-100
+ const baseY = 101 * TILE;
+ // shadow
+ ctx.fillStyle = "rgba(0,0,0,0.18)";
+ ctx.beginPath();
+ ctx.ellipse(cx, baseY - 4, 26, 7, 0, 0, Math.PI * 2);
+ ctx.fill();
+ // tapered tower with stripes
+ const towerH = TILE * 2.6;
+ const topY = baseY - towerH;
+ ctx.beginPath();
+ ctx.moveTo(cx - 20, baseY - 6);
+ ctx.lineTo(cx - 11, topY + 14);
+ ctx.lineTo(cx + 11, topY + 14);
+ ctx.lineTo(cx + 20, baseY - 6);
+ ctx.closePath();
+ ctx.fillStyle = "#e8e0d0";
+ ctx.fill();
+ ctx.save();
+ ctx.clip();
+ ctx.fillStyle = "#c9524a";
+ for (let i = 0; i < 3; i++) ctx.fillRect(cx - 22, topY + 20 + i * 22, 44, 10);
+ ctx.restore();
+ // lamp room
+ ctx.fillStyle = "#2a3550";
+ ctx.fillRect(cx - 12, topY, 24, 16);
+ const pulse = 0.55 + 0.45 * Math.sin(time * 2.4);
+ ctx.fillStyle = `rgba(255, 228, 100, ${pulse})`;
+ ctx.fillRect(cx - 8, topY + 3, 16, 10);
+ ctx.fillStyle = "#c9524a";
+ ctx.beginPath();
+ ctx.moveTo(cx - 14, topY);
+ ctx.lineTo(cx, topY - 12);
+ ctx.lineTo(cx + 14, topY);
+ ctx.closePath();
+ ctx.fill();
+ // the sweeping beam
+ const ang = time * 0.8;
+ const bx = Math.cos(ang);
+ const by = Math.sin(ang) * 0.45; // flattened sweep
+ ctx.fillStyle = `rgba(255, 235, 140, ${0.10 + 0.06 * pulse})`;
+ ctx.beginPath();
+ ctx.moveTo(cx, topY + 8);
+ ctx.lineTo(cx + bx * 150 - by * 26, topY + 8 + by * 150 + bx * 12);
+ ctx.lineTo(cx + bx * 150 + by * 26, topY + 8 + by * 150 - bx * 12);
+ ctx.closePath();
+ ctx.fill();
+ // plaque
+ ctx.font = "bold 10.5px 'Trebuchet MS', sans-serif";
+ const label = "㉚ HORIZON BEACON — press E";
+ const tw = ctx.measureText(label).width + 14;
+ ctx.fillStyle = "#1d3357";
+ this.roundedRect(ctx, cx - tw / 2, baseY + 2, tw, 15, 4);
+ ctx.fill();
+ ctx.fillStyle = "#ffe44d";
+ ctx.textAlign = "center";
+ ctx.textBaseline = "middle";
+ ctx.fillText(label, cx, baseY + 9.5);
+ ctx.textBaseline = "alphabetic";
+ }
+
/** the tour kiosk: a signpost with the ① disc — stand next to it and
* press E to ride the express to the first stop */
private drawKiosk(ctx: CanvasRenderingContext2D, time: number): void {
@@ -562,9 +702,23 @@ export class City {
for (let x = 60; x >= 6; x -= 3) chevron(x * TILE, 51 * TILE, -1, 0);
// down the west edge to Language Lane
for (let y = 52.5; y <= 60; y += 2.5) chevron(3 * TILE, y * TILE, 0, 1);
- // stops 19→21: east along the frontier's south road — the route ends
- // at Echo Tower's door, so the arrows stop there too
+ // stops 19→21: east along the frontier's south road
for (let x = 5; x <= 46; x += 3) chevron(x * TILE, 62 * TILE, 1, 0);
+ // …then back to the center junction and south to the ferry pier
+ for (let y = 63.5; y <= 66.5; y += 1.5) chevron(31 * TILE, y * TILE, 0, 1);
+ // (the boat crosses the open water — no arrows on the sea)
+ // island pier landing → down the center to the Kernel Shore road
+ for (let y = 75; y <= 85; y += 2.5) chevron(31 * TILE, y * TILE, 0, 1);
+ // two-lane road: west (bottom lane) to stop ㉒, then east (top lane)
+ // through ㉓ ㉔ ㉕ to the east junction
+ for (let x = 28; x >= 9; x -= 3) chevron(x * TILE, 87.5 * TILE, -1, 0);
+ for (let x = 8; x <= 58; x += 3) chevron(x * TILE, 86.45 * TILE, 1, 0);
+ // down the east edge to Assembly Row
+ for (let y = 89; y <= 96; y += 2.5) chevron(61 * TILE, y * TILE, 0, 1);
+ // west along the south road: stops ㉖ → ㉙
+ for (let x = 58; x >= 9; x -= 3) chevron(x * TILE, 98 * TILE, -1, 0);
+ // and down to the Horizon Beacon — the end of the whole tour
+ chevron(9 * TILE, 99.3 * TILE, 0, 1);
}
/** the current mini-batch riding the training loop: little chips carrying
diff --git a/src/world/game.ts b/src/world/game.ts
index 65f2f74..c00ecab 100644
--- a/src/world/game.ts
+++ b/src/world/game.ts
@@ -26,8 +26,16 @@ export class Game {
blurbEl: HTMLElement;
time = 0;
private last = performance.now();
- /** the tour express: a short scripted flight from the kiosk to stop ① */
- private travel: { pts: { x: number; y: number }[]; lens: number[]; total: number; t: number; dur: number } | null = null;
+ /** the tour express (kiosk → stop ①) and the ferry both reuse this: a
+ * scripted glide along waypoints; "boat" mode draws the ferry hull */
+ private travel: {
+ pts: { x: number; y: number }[];
+ lens: number[];
+ total: number;
+ t: number;
+ dur: number;
+ mode?: "walk" | "boat";
+ } | null = null;
private trail: { x: number; y: number; age: number }[] = [];
/** disable movement while a panel is open */
get inputLocked(): boolean {
@@ -83,6 +91,15 @@ export class Game {
this.startTourExpress();
return;
}
+ const ferry = this.city.ferryAt(tx, ty);
+ if (ferry) {
+ this.startFerry(ferry);
+ return;
+ }
+ if (this.city.beaconNear(tx, ty)) {
+ openPanel("beacon.journey", this.world);
+ return;
+ }
const b = this.city.doorAt(tx, ty);
if (b) {
this.enterBuilding(b);
@@ -142,7 +159,37 @@ export class Game {
lens.push(L);
total += L;
}
- this.travel = { pts, lens, total, t: 0, dur: Math.max(1.4, total / 420) };
+ this.travel = { pts, lens, total, t: 0, dur: Math.max(1.4, total / 420), mode: "walk" };
+ }
+
+ /** the ferry: a slow boat ride between the mainland pier and the island
+ * pier — the only way across the sea */
+ startFerry(dest: "island" | "mainland"): void {
+ if (this.scene.kind !== "city" || this.travel) return;
+ const cx = 31 * TILE; // between the pier's two tile columns
+ const pts =
+ dest === "island"
+ ? [
+ { x: this.avatar.x, y: this.avatar.y },
+ { x: cx, y: 67.4 * TILE },
+ { x: cx, y: 72.6 * TILE },
+ { x: cx, y: 74.5 * TILE },
+ ]
+ : [
+ { x: this.avatar.x, y: this.avatar.y },
+ { x: cx, y: 72.6 * TILE },
+ { x: cx, y: 67.4 * TILE },
+ { x: cx, y: 65.5 * TILE },
+ ];
+ const lens: number[] = [];
+ let total = 0;
+ for (let i = 1; i < pts.length; i++) {
+ const L = Math.hypot(pts[i].x - pts[i - 1].x, pts[i].y - pts[i - 1].y);
+ lens.push(L);
+ total += L;
+ }
+ // boats are slower than the express — let the crossing feel like one
+ this.travel = { pts, lens, total, t: 0, dur: Math.max(2.2, total / 180), mode: "boat" };
}
private updateTravel(dt: number): void {
@@ -240,6 +287,7 @@ export class Game {
ctx.arc(p.x, p.y, 10, 0, Math.PI * 2);
ctx.fill();
}
+ if (this.travel?.mode === "boat") this.drawFerryBoat(ctx);
this.avatar.draw(ctx, this.time);
ctx.restore();
} else {
@@ -268,6 +316,41 @@ export class Game {
}
}
+ /** the little ferry hull, drawn under the avatar while crossing the sea */
+ private drawFerryBoat(ctx: CanvasRenderingContext2D): void {
+ const x = this.avatar.x;
+ const y = this.avatar.y + 8;
+ const bob = Math.sin(this.time * 3) * 1.5;
+ // wake
+ ctx.fillStyle = "rgba(255,255,255,0.25)";
+ ctx.beginPath();
+ ctx.ellipse(x, y + 10 + bob, 24, 6, 0, 0, Math.PI * 2);
+ ctx.fill();
+ // hull
+ ctx.fillStyle = "#7a4e2d";
+ ctx.beginPath();
+ ctx.moveTo(x - 22, y - 4 + bob);
+ ctx.quadraticCurveTo(x, y + 16 + bob, x + 22, y - 4 + bob);
+ ctx.closePath();
+ ctx.fill();
+ ctx.fillStyle = "#9a6a3f";
+ ctx.fillRect(x - 20, y - 8 + bob, 40, 6);
+ // stern flag
+ ctx.strokeStyle = "#5a3a20";
+ ctx.lineWidth = 2;
+ ctx.beginPath();
+ ctx.moveTo(x + 16, y - 8 + bob);
+ ctx.lineTo(x + 16, y - 22 + bob);
+ ctx.stroke();
+ ctx.fillStyle = "#ffd34d";
+ ctx.beginPath();
+ ctx.moveTo(x + 16, y - 22 + bob);
+ ctx.lineTo(x + 26, y - 18 + bob);
+ ctx.lineTo(x + 16, y - 14 + bob);
+ ctx.closePath();
+ ctx.fill();
+ }
+
private updatePrompt(): void {
let prompt = "";
let blurb = "";
@@ -282,7 +365,21 @@ export class Game {
} else if (this.city.kioskNear(tx, ty)) {
prompt = "Press E — ride the express to tour stop ①";
blurb =
- "The guided tour starts at the Dataset Warehouse in the north-west. Stops ① → ㉑ trace the whole story: data → forward → loss → backward → step → metrics — then across the river, where the data stops being images.";
+ "The guided tour starts at the Dataset Warehouse in the north-west. Stops ① → ㉚ trace the whole story: data → forward → loss → backward → step → metrics — then across the river, where the data stops being images, and finally over the sea by ferry to the Foundations Isle, where everything gets built from scratch.";
+ } else if (this.city.ferryAt(tx, ty)) {
+ const dest = this.city.ferryAt(tx, ty)!;
+ prompt =
+ dest === "island"
+ ? "Press E — take the ferry to the Foundations Isle"
+ : "Press E — take the ferry back to the Frontier";
+ blurb =
+ dest === "island"
+ ? "Across the water lies the Foundations Isle: convolutions that learn their own kernels, residual networks, an optimizer race, class-activation lighthouse beams — and the training loop itself, rebuilt from scratch. Stops ㉒ → ㉚."
+ : "The ferry back to the mainland: the Frontier's ratings, trees and language models, and beyond them the old town where the whole story began.";
+ } else if (this.city.beaconNear(tx, ty)) {
+ prompt = "Press E — read the Horizon Beacon";
+ blurb =
+ "The last stop of the tour. The beacon looks back over everything the city trains — and out at the open water, where the map ends and your own projects begin.";
} else if (this.city.chipNear(this.avatar.x, this.avatar.y)) {
const running = this.world.main.running;
const batchSize = this.world.mlp.lastBatch.length;
diff --git a/tests/engine.test.ts b/tests/engine.test.ts
index fc0f62e..a6c538c 100644
--- a/tests/engine.test.ts
+++ b/tests/engine.test.ts
@@ -4,6 +4,7 @@ import {
add,
addRow,
bceWithLogits,
+ concatCols,
crossEntropy,
gatherCols,
matmul,
@@ -16,7 +17,15 @@ import {
softmax,
sub,
} from "../src/engine/tensor";
-import { SGD } from "../src/engine/optim";
+import { Adam, Momentum, RMSProp, SGD } from "../src/engine/optim";
+import {
+ avgPoolAll,
+ batchNorm,
+ conv2d,
+ convolveImage,
+ im2col,
+ makeBnStats,
+} from "../src/engine/conv";
import { mulberry32 } from "../src/engine/data";
/**
@@ -134,6 +143,12 @@ describe("gradients vs numeric differentiation", () => {
gradCheck((inp) => mseLoss(inp[0], inp[1]), [preds, targets]);
});
+ it("concatCols joins features and splits gradients", () => {
+ const a = Tensor.randn([3, 2], 1, rand);
+ const b = Tensor.randn([3, 4], 1, rand);
+ gradCheck((inp) => mean(mul(concatCols(inp[0], inp[1]), concatCols(inp[0], inp[1]))), [a, b]);
+ });
+
it("gatherCols routes gradients to picked elements only", () => {
const a = Tensor.randn([3, 4], 1, rand);
a.requiresGrad = true;
@@ -146,6 +161,154 @@ describe("gradients vs numeric differentiation", () => {
});
});
+describe("convolution ops", () => {
+ it("conv2d matches a hand-computed 3×3 kernel on a 4×4 image", () => {
+ // one image, one channel, identity-ish check against convolveImage
+ const img = Float32Array.from({ length: 16 }, (_, i) => i + 1);
+ const kernel = Float32Array.from([-1, -1, -1, 0, 0, 0, 1, 1, 1]); // top edge
+ const x = new Tensor(img.slice(), [1, 16]);
+ const w = new Tensor(kernel.slice(), [1, 9]);
+ const y = conv2d(x, w, null, { cin: 1, h: 4, w: 4, k: 3, stride: 1, pad: 0 });
+ expect(y.shape).toEqual([1, 4]); // 2×2 map
+ const ref = convolveImage(img, 4, 4, kernel, 3, 1, 0);
+ expect(Array.from(y.data)).toEqual(Array.from(ref.data));
+ // and the top-left window by hand: -(1+2+3) + (9+10+11) = 24
+ expect(y.data[0]).toBeCloseTo(24, 5);
+ });
+
+ it("conv2d respects stride and padding sizes", () => {
+ const x = Tensor.randn([2, 2 * 5 * 5], 1, rand);
+ const w = Tensor.randn([3, 2 * 3 * 3], 1, rand);
+ const b = Tensor.randn([3], 1, rand);
+ const y = conv2d(x, w, b, { cin: 2, h: 5, w: 5, k: 3, stride: 2, pad: 1 });
+ // (5 + 2*1 - 3)//2 + 1 = 3
+ expect(y.shape).toEqual([2, 3 * 3 * 3]);
+ });
+
+ it("conv2d gradients check out (x, weight and bias)", () => {
+ const x = Tensor.randn([2, 1 * 4 * 4], 1, rand);
+ const w = Tensor.randn([2, 1 * 3 * 3], 0.5, rand);
+ const b = Tensor.randn([2], 0.5, rand);
+ gradCheck(
+ (inp) =>
+ mean(conv2d(inp[0], inp[1], inp[2], { cin: 1, h: 4, w: 4, k: 3, stride: 2, pad: 1 })),
+ [x, w, b],
+ );
+ });
+
+ it("conv2d = im2col windows @ kernel (the conv-as-matmul view)", () => {
+ const img = new Float32Array(25);
+ const r = mulberry32(3);
+ for (let i = 0; i < 25; i++) img[i] = r();
+ const kernel = Float32Array.from([0, -1, 1, -1, 1, 0, 1, 0, 0]);
+ const direct = convolveImage(img, 5, 5, kernel, 3);
+ const { cols, rows, colsPerRow } = im2col(img, 5, 5, 3);
+ for (let row = 0; row < rows; row++) {
+ let s = 0;
+ for (let j = 0; j < colsPerRow; j++) s += cols[row * colsPerRow + j] * kernel[j];
+ expect(s).toBeCloseTo(direct.data[row], 5);
+ }
+ });
+
+ it("avgPoolAll averages each channel and routes gradients evenly", () => {
+ const x = Tensor.fromArray([1, 2, 3, 4, 10, 20, 30, 40], [1, 8]); // 2 channels × 4 cells
+ const y = avgPoolAll(x, 2);
+ expect(Array.from(y.data)).toEqual([2.5, 25]);
+ const xg = Tensor.randn([3, 2 * 4], 1, rand);
+ gradCheck((inp) => mean(avgPoolAll(inp[0], 2)), [xg]);
+ });
+
+ it("batchNorm normalizes per channel and its gradients check out", () => {
+ const x = Tensor.randn([4, 2 * 3], 2, rand);
+ const gamma = Tensor.fromArray([1, 1]);
+ const beta = Tensor.fromArray([0, 0]);
+ const y = batchNorm(x, gamma, beta, 2, null, true);
+ // per-channel mean ≈ 0, std ≈ 1
+ for (let ch = 0; ch < 2; ch++) {
+ let m = 0;
+ for (let b = 0; b < 4; b++) for (let i = 0; i < 3; i++) m += y.data[b * 6 + ch * 3 + i];
+ m /= 12;
+ expect(Math.abs(m)).toBeLessThan(1e-5);
+ }
+ const xg = Tensor.randn([3, 2 * 2], 1, rand);
+ const gg = Tensor.randn([2], 0.5, rand);
+ const bg = Tensor.randn([2], 0.5, rand);
+ gradCheck((inp) => mean(mul(batchNorm(inp[0], inp[1], inp[2], 2, null, true), inp[0])), [
+ xg,
+ gg,
+ bg,
+ ]);
+ });
+
+ it("batchNorm eval mode uses running stats", () => {
+ const stats = makeBnStats(1, 1); // momentum 1: running stats = last batch stats
+ const gamma = Tensor.fromArray([2]);
+ const beta = Tensor.fromArray([1]);
+ const xTrain = Tensor.fromArray([0, 2, 4, 6], [4, 1]);
+ batchNorm(xTrain, gamma, beta, 1, stats, true);
+ expect(stats.runMean[0]).toBeCloseTo(3, 5);
+ const xEval = Tensor.fromArray([3], [1, 1]);
+ const yEval = batchNorm(xEval, gamma, beta, 1, stats, false);
+ // (3-3)/std*2 + 1 = 1
+ expect(yEval.data[0]).toBeCloseTo(1, 4);
+ });
+
+ it("batchNorm eval-mode gradients check out (affine with fixed running stats)", () => {
+ // eval mode is a different backward path from training: mean/var are
+ // constants (the running stats), so it's a plain affine map. Fix
+ // non-trivial running stats and gradcheck x, gamma and beta through it.
+ // (eval mode never updates the stats, so repeated gradCheck calls stay pure)
+ const stats = makeBnStats(2, 0.1);
+ stats.runMean.set([0.3, -0.2]);
+ stats.runVar.set([0.5, 1.5]);
+ const xg = Tensor.randn([3, 2 * 2], 1, rand);
+ const gg = Tensor.randn([2], 0.5, rand);
+ const bg = Tensor.randn([2], 0.5, rand);
+ gradCheck(
+ (inp) => mean(mul(batchNorm(inp[0], inp[1], inp[2], 2, stats, false), inp[0])),
+ [xg, gg, bg],
+ );
+ });
+});
+
+describe("optimizers", () => {
+ const quadratic = (opt: (p: Tensor) => { step(): void; zeroGrad(): void }) => {
+ // minimize (p - 3)² elementwise from p = 0
+ const p = Tensor.zeros([4], true);
+ const o = opt(p);
+ for (let s = 0; s < 200; s++) {
+ o.zeroGrad();
+ p.ensureGrad();
+ for (let i = 0; i < 4; i++) p.grad![i] = 2 * (p.data[i] - 3);
+ o.step();
+ }
+ return p.data;
+ };
+
+ it("momentum, RMSProp and Adam all reach the minimum of a quadratic", () => {
+ for (const make of [
+ (p: Tensor) => new Momentum([p], 0.02, 0.9),
+ (p: Tensor) => new RMSProp([p], 0.05),
+ (p: Tensor) => new Adam([p], 0.1),
+ ]) {
+ const data = quadratic(make);
+ for (let i = 0; i < 4; i++) expect(Math.abs(data[i] - 3)).toBeLessThan(0.05);
+ }
+ });
+
+ it("Adam keeps the moving averages the book describes", () => {
+ const p = Tensor.zeros([1], true);
+ const adam = new Adam([p], 0.01, 0.9, 0.99);
+ p.ensureGrad()[0] = 1;
+ adam.step();
+ // after one step: avg = (1-beta1)*g = 0.1, sqr = (1-beta2)*g² = 0.01
+ expect(adam.avg[0][0]).toBeCloseTo(0.1, 6);
+ expect(adam.sqrAvg[0][0]).toBeCloseTo(0.01, 6);
+ // unbias divisor after the step counted: 1 - 0.9¹ = 0.1 → unbiased avg = 1
+ expect(adam.avg[0][0] / (1 - Math.pow(0.9, 1))).toBeCloseTo(1, 6);
+ });
+});
+
describe("end-to-end learning", () => {
it("linear model learns a separable 2D problem with SGD", () => {
// points above the line y=x get label 1
diff --git a/tests/island.test.ts b/tests/island.test.ts
new file mode 100644
index 0000000..bbb3329
--- /dev/null
+++ b/tests/island.test.ts
@@ -0,0 +1,245 @@
+// The Foundations Isle's scenarios: the CNN, the residual race, the
+// optimizer derby, the siamese pair-matcher, and the from-scratch Learner
+// with its callback system. These are smoke + property tests: real training
+// should reduce the loss, and the load-bearing tricks (identity blocks, the
+// NaN guard, transfer) should behave as the book describes.
+
+import { readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+import { MnistData } from "../src/engine/data";
+import { ConvNet, OptRace, ResRace, SiamesePairs } from "../src/sim/scenarios3";
+import { LearnerLab, MiniLearner, TerminateOnNaN } from "../src/sim/learner";
+
+function loadMnistFromDisk(): MnistData {
+ const here = dirname(fileURLToPath(import.meta.url));
+ const buf = new Uint8Array(readFileSync(join(here, "..", "public", "data", "mnist.bin")));
+ const view = new DataView(buf.buffer, buf.byteOffset);
+ const nTrain = view.getUint32(4, true);
+ const nTest = view.getUint32(8, true);
+ let off = 12;
+ const trainImages = buf.subarray(off, (off += nTrain * 784));
+ const trainLabels = buf.subarray(off, (off += nTrain));
+ const testImages = buf.subarray(off, (off += nTest * 784));
+ const testLabels = buf.subarray(off, (off += nTest));
+ return { trainImages, trainLabels, testImages, testLabels, nTrain, nTest };
+}
+
+const data = loadMnistFromDisk();
+
+/** run a scenario a few steps and return first/last loss */
+function trainA(sc: { trainStep(): number }, steps: number): { first: number; last: number } {
+ let first = NaN;
+ let last = NaN;
+ for (let i = 0; i < steps; i++) {
+ const l = sc.trainStep();
+ if (i === 0) first = l;
+ last = l;
+ }
+ return { first, last };
+}
+
+describe("Convolution Works (ConvNet)", () => {
+ it("produces finite losses and 10 logits per image", () => {
+ const cnn = new ConvNet(data);
+ const { first, last } = trainA(cnn, 12);
+ expect(isFinite(first)).toBe(true);
+ expect(isFinite(last)).toBe(true);
+ // one warm step already ran in the constructor path? not here — check inference shape
+ const img = new Float32Array(784);
+ for (let j = 0; j < 784; j++) img[j] = data.testImages[j] / 255;
+ const { logits, probs } = cnn.inferMaps(img);
+ expect(logits.length).toBe(10);
+ let psum = 0;
+ for (const p of probs) psum += p;
+ expect(psum).toBeCloseTo(1, 4);
+ });
+
+ it("records activation statistics and can toggle batchnorm without crashing", () => {
+ const cnn = new ConvNet(data);
+ cnn.useBN = true;
+ trainA(cnn, 6);
+ expect(cnn.actStats.length).toBeGreaterThan(0);
+ const last = cnn.actStats.at(-1)!;
+ expect(last.nearZero).toBeGreaterThanOrEqual(0);
+ expect(last.nearZero).toBeLessThanOrEqual(1);
+ // eval-mode forward (uses running stats) must stay finite
+ expect(isFinite(cnn.evaluate())).toBe(true);
+ });
+
+ it("reduces its training loss over a short run", () => {
+ // small batch keeps each heavy conv step quick; average early vs late
+ // loss to smooth out batch noise
+ const cnn = new ConvNet(data, 0.1, 12);
+ let early = 0;
+ for (let i = 0; i < 8; i++) early += cnn.trainStep();
+ for (let i = 0; i < 24; i++) cnn.trainStep();
+ let late = 0;
+ for (let i = 0; i < 8; i++) late += cnn.trainStep();
+ expect(late / 8).toBeLessThan(early / 8);
+ }, 30000);
+});
+
+describe("Residual Works (ResRace)", () => {
+ it("each block is an exact identity at initialization", () => {
+ const r = new ResRace(data);
+ // at init the second conv of each block is zero → f(x)=0 → ratios ≈ 0
+ r.trainStep(); // one step records the display graph
+ // rebuild a clean display pass by inspecting a fresh forward's nodes:
+ // easier — check the raw residual ratios right after the recorded step.
+ // (After a step the convs have moved a hair, so allow a small tolerance.)
+ const ratios = r.residualRatios();
+ for (const v of ratios) expect(v).toBeLessThan(0.5);
+ });
+
+ it("trains both twins and keeps them finite", () => {
+ const r = new ResRace(data, 0.1);
+ const { first, last } = trainA(r, 15);
+ expect(isFinite(first)).toBe(true);
+ expect(isFinite(last)).toBe(true);
+ expect(isFinite(r.lastPlainLoss)).toBe(true);
+ expect(isFinite(r.accuracy(true))).toBe(true);
+ expect(isFinite(r.accuracy(false))).toBe(true);
+ });
+
+ it("CAM and Grad-CAM produce finite 7×7 heatmaps", () => {
+ const r = new ResRace(data);
+ trainA(r, 5);
+ const img = r.batchX14(Int32Array.from([0]), data.testImages);
+ // pull the 14×14 pixels back out for the cam helpers
+ const px = img.data.slice(0, 196);
+ const { cam } = r.camFor(px, 3);
+ expect(cam.length).toBe(49);
+ expect(cam.every((v) => isFinite(v))).toBe(true);
+ const gcam = r.gradCamFor(px, 3, 1);
+ expect(gcam.length).toBe(49);
+ expect(gcam.every((v) => isFinite(v))).toBe(true);
+ });
+
+ it("runs on a different input size (fully convolutional)", () => {
+ const r = new ResRace(data);
+ const full = new Float32Array(784);
+ for (let j = 0; j < 784; j++) full[j] = data.testImages[j] / 255;
+ const out = r.runAnySize(full, 28);
+ expect(out.logits.length).toBe(10);
+ expect(out.logits.every((v) => isFinite(v))).toBe(true);
+ });
+});
+
+describe("Optimizer Institute (OptRace)", () => {
+ it("keeps four independent racers with their own optimizer state", () => {
+ const race = new OptRace(data, 0.1);
+ trainA(race, 20);
+ expect(race.racers.map((r) => r.name)).toEqual(["SGD", "Momentum", "RMSProp", "Adam"]);
+ // every racer has a finite loss history and its own accuracy
+ for (const r of race.racers) {
+ expect(r.lossHistory.length).toBeGreaterThan(0);
+ expect(isFinite(race.accuracyOf(r))).toBe(true);
+ }
+ // the momentum desk recorded the watched parameter, read from the live
+ // optimizer state (not a re-derivation) — its avg must equal the racer's
+ // actual gradAvg for the watched element
+ expect(race.watched.length).toBeGreaterThan(0);
+ const momRacer = race.racers.find((r) => r.name === "Momentum")!;
+ const momOpt = momRacer.opt as import("../src/engine/optim").Momentum;
+ const pi = momOpt.params.indexOf(momRacer.b2);
+ expect(race.watched.at(-1)!.avg).toBe(momOpt.gradAvg[pi][3]);
+ // Adam's bookkeeping is populated and the step counter advanced
+ const st = race.adamState(3);
+ expect(st.step).toBeGreaterThan(0);
+ expect(isFinite(st.unbiased)).toBe(true);
+ });
+
+ it("one lr dial fans out to all four optimizers", () => {
+ const race = new OptRace(data);
+ race.opt.lr = 0.033;
+ for (const r of race.racers) expect(r.opt.lr).toBeCloseTo(0.033, 6);
+ });
+});
+
+describe("Architecture Atelier (SiamesePairs)", () => {
+ it("transfers the convnet body and freezes/unfreezes the encoder", () => {
+ const cnn = new ConvNet(data);
+ trainA(cnn, 3);
+ const siam = new SiamesePairs(data);
+ expect(siam.encoderSource).toBe("random init");
+ siam.transferFrom(cnn);
+ expect(siam.encoderSource).toContain("cut from");
+ // the encoder weights now match the convnet's first two layers
+ expect(siam.enc1.w.data[0]).toBe(cnn.layers[0].w.data[0]);
+ // freeze → encoder group lr is 0; unfreeze restores it
+ siam.encLr = 0.04;
+ siam.setFrozen(true);
+ expect(siam.opt.groups[0].lr).toBe(0);
+ siam.setFrozen(false);
+ expect(siam.opt.groups[0].lr).toBeCloseTo(0.04, 6);
+ });
+
+ it("trains the pair-matcher to finite loss and valid probabilities", () => {
+ const siam = new SiamesePairs(data);
+ const { first, last } = trainA(siam, 15);
+ expect(isFinite(first)).toBe(true);
+ expect(isFinite(last)).toBe(true);
+ const { pSame } = siam.classifyPair(0, 1);
+ expect(pSame).toBeGreaterThanOrEqual(0);
+ expect(pSame).toBeLessThanOrEqual(1);
+ expect(isFinite(siam.evaluate())).toBe(true);
+ });
+});
+
+describe("Learner Hall (MiniLearner + callbacks)", () => {
+ it("fires the loop events in order and counts them", () => {
+ const m = new MiniLearner(data);
+ m.oneBatch();
+ // a full batch fires the whole ladder from before_fit through after_batch
+ expect(m.eventCounts["before_fit"]).toBe(1);
+ expect(m.eventCounts["after_pred"]).toBeGreaterThan(0);
+ expect(m.eventCounts["after_step"]).toBeGreaterThan(0);
+ expect(m.eventCounts["after_batch"]).toBeGreaterThan(0);
+ expect(m.stepsDone).toBe(1);
+ });
+
+ it("TerminateOnNaN raises CancelFitException and halts the fit", () => {
+ const m = new MiniLearner(data);
+ // make sure the guard is on, then blow up the learning rate. The engine's
+ // stable softmax bounds cross-entropy until the *weights* overflow
+ // float32, which needs an absurd lr — that's exactly the point of the
+ // Callback Rack's "overflow the weights" button.
+ m.cbs.find((c) => c instanceof TerminateOnNaN)!.enabled = true;
+ m.opt.lr = 1e20;
+ for (let i = 0; i < 20 && !m.cancelled; i++) m.oneBatch();
+ expect(m.cancelled).not.toBeNull();
+ // attribution comes from the callback that actually threw (stamped in call()),
+ // not a hardcoded lookup
+ expect(m.cancelled!.by).toBe("TerminateOnNaN");
+ // after cancellation, after_fit ran and further batches are no-ops
+ const stepsAtCancel = m.stepsDone;
+ m.oneBatch();
+ expect(m.stepsDone).toBe(stepsAtCancel);
+ // reset recovers AND restarts the loop bookkeeping (not just the weights)
+ m.reset();
+ expect(m.cancelled).toBeNull();
+ expect(m.stepsDone).toBe(0);
+ expect(m.epochsDone).toBe(0);
+ expect(m.eventCounts["after_batch"]).toBe(0);
+ expect(m.lastTrace).toEqual([]);
+ });
+
+ it("without the guard, a diverging run silently produces NaN (the ablation)", () => {
+ const m = new MiniLearner(data);
+ m.cbs.find((c) => c instanceof TerminateOnNaN)!.enabled = false;
+ m.opt.lr = 1e20;
+ for (let i = 0; i < 20; i++) m.oneBatch();
+ expect(m.cancelled).toBeNull(); // nothing stopped it
+ expect(isFinite(m.lastLossValue)).toBe(false); // and the loss is junk
+ });
+
+ it("the LearnerLab scenario wraps the learner for the world's Trainer", () => {
+ const lab = new LearnerLab(data);
+ const { first } = trainA(lab, 8);
+ expect(isFinite(first)).toBe(true);
+ expect(lab.step).toBe(8);
+ expect(isFinite(lab.evaluate())).toBe(true);
+ });
+});