diff --git a/.factory/library/architecture.md b/.factory/library/architecture.md index 00d322f..ee0c0b0 100644 --- a/.factory/library/architecture.md +++ b/.factory/library/architecture.md @@ -64,6 +64,7 @@ Unified event system replacing EventBus + CustomEvents. Topics with `equals: fal - Bus topics created: `ml.outputs`, `ml.outputCount`, `mode.output` - Debug probe (`probe/debug-probe.ts`) now wraps MLStore instead of raw WasmIML - `window.__nispsStore` exposed for test access to store methods like `setOutputMode()` +- `_updateOutputs(Float32Array)` — internal method for direct output signal update (used by HeatmapStrip drag-to-set) ### Input Store (`stores/input-store.ts`) — IMPLEMENTED - Factory function `createInputStore(mlStore, bus)` returns `InputStore` @@ -93,6 +94,19 @@ Unified event system replacing EventBus + CustomEvents. Topics with `equals: fal - Exposed as `#flowfield-canvas` for e2e test targeting - Mounted in App.tsx when `ready()` signal is true +### HeatmapStrip Component (`components/visual/HeatmapStrip.tsx`) — IMPLEMENTED +- Fixed-position bar chart strip at top of viewport (height 22px, z-index 20) +- One bar per ML output; bar width proportional to output value (0-100%) +- Bar count matches current output mode: visual=20, synth=126, midi-cc=8, audio-canvas=36 +- Reactively rebuilds when `outputCount()` changes (mode switch triggers `` re-render) +- Visual mode uses curated colors/names (VISUAL_PARAM_COLORS/VISUAL_PARAM_NAMES); other modes use hue-based color generation +- Drag-to-set: pointer capture on cell, drag updates output value directly via `iml.setOutput()` + `_updateOutputs()` +- Click-to-popup: short click (no drag) opens param popup (name, value, color bar, close button) +- Tooltip on hover: shows "ParamName: value" positioned below hovered cell +- Per-cell event handler factory `createCellHandlers()` captures element reference for pointer events +- ML store exposes `_updateOutputs(Float32Array)` for direct output signal update from drag +- CSS: glass morphism strip, per-bar colored fill, brightness filter on hover, drag cursor + ### Output Store (`stores/output-store.ts`) — NOT YET IMPLEMENTED - Output mode (visual/synth/midi-cc/audio-canvas) - Per-mode overrides arrays diff --git a/src/App.tsx b/src/App.tsx index ebb3c02..3e4ceb4 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -5,6 +5,7 @@ import { exposeDebugProbe } from './probe/debug-probe'; import bus from './bus'; import Joystick from './components/input/Joystick'; import FlowField from './components/visual/FlowField'; +import HeatmapStrip from './components/visual/HeatmapStrip'; import Dock from './components/layout/Dock'; import TrainingDrawer from './components/layout/TrainingDrawer'; import ModeDrawer from './components/layout/ModeDrawer'; @@ -137,6 +138,7 @@ export default function App() { {/* Dock + Drawers + Status (only when ready and ML store exists) */} +
(null); + + // ─── Helpers ─── + + function getParamNames(): string[] { + const mode = props.mlStore.state.outputMode; + const count = props.mlStore.outputCount(); + if (mode === 'visual') return VISUAL_PARAM_NAMES; + return Array.from({ length: count }, (_, i) => `p${i}`); + } + + function getParamColors(): string[] { + const mode = props.mlStore.state.outputMode; + const count = props.mlStore.outputCount(); + if (mode === 'visual') return VISUAL_PARAM_COLORS; + return Array.from({ length: count }, (_, i) => defaultParamColor(i)); + } + + function getTooltipContent(paramIndex: number): string { + const names = getParamNames(); + const outputs = props.mlStore.outputs(); + const name = names[paramIndex] ?? `p${paramIndex}`; + const value = outputs[paramIndex]?.toFixed(3) ?? '—'; + return `${name}: ${value}`; + } + + // ─── Apply drag value ─── + + function applyDragValue(paramIndex: number, clientX: number, cellEl: HTMLElement) { + const rect = cellEl.getBoundingClientRect(); + const x = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); + + // Directly set the output in the IML (bypass inference, set output directly) + const iml = props.mlStore.getActiveIml(); + if (iml) { + iml.setOutput(paramIndex, x); + const newOutputs = new Float32Array(iml.getOutputs()); + (props.mlStore as any)._updateOutputs(newOutputs); + } + + // Update tooltip + const names = getParamNames(); + const name = names[paramIndex] ?? `p${paramIndex}`; + setTooltipText(`${name}: ${x.toFixed(3)}`); + setTooltipVisible(true); + setTooltipLeft(rect.left); + } + + // ─── Per-cell event handlers (matching old app's pattern) ─── + + function createCellHandlers(paramIndex: number) { + let downX = 0; + let downY = 0; + let didDrag = false; + let isDragging = false; + // Capture cell element reference for use in move/up where e.currentTarget may be null + // (e.g. when events are dispatched programmatically) + let cellEl: HTMLElement | null = null; + + function onPointerDown(e: PointerEvent) { + e.preventDefault(); + cellEl = (e.currentTarget as HTMLElement) || cellEl; + if (cellEl) cellEl.setPointerCapture(e.pointerId); + downX = e.clientX; + downY = e.clientY; + didDrag = false; + isDragging = true; + if (cellEl) cellEl.classList.add('dragging'); + } + + function onPointerMove(e: PointerEvent) { + if (!isDragging || !cellEl) return; + const dx = Math.abs(e.clientX - downX); + const dy = Math.abs(e.clientY - downY); + if (dx > 3 || dy > 3) didDrag = true; + if (didDrag) { + applyDragValue(paramIndex, e.clientX, cellEl); + } + } + + function onPointerUp(e: PointerEvent) { + if (!isDragging || !cellEl) return; + cellEl.classList.remove('dragging'); + isDragging = false; + + if (!didDrag) { + // Short click — open param popup + const rect = cellEl.getBoundingClientRect(); + setPopup({ paramIndex, cellRect: rect }); + } + } + + function onPointerCancel() { + if (isDragging && cellEl) { + cellEl.classList.remove('dragging'); + } + isDragging = false; + } + + function onPointerEnter(e: PointerEvent) { + if (isDragging) return; + const el = (e.currentTarget as HTMLElement) || cellEl; + if (!el) return; + const rect = el.getBoundingClientRect(); + setTooltipText(getTooltipContent(paramIndex)); + setTooltipVisible(true); + setTooltipLeft(rect.left); + } + + function onPointerLeave() { + if (!isDragging) { + setTooltipVisible(false); + } + } + + function setRef(el: HTMLElement) { + cellEl = el; + } + + return { + setRef, + onPointerDown, + onPointerMove, + onPointerUp, + onPointerCancel, + onPointerEnter, + onPointerLeave, + }; + } + + // ─── Close popup ─── + + function closePopup() { + setPopup(null); + } + + // ─── Render ─── + + // Derive cell data reactively from outputCount + outputs + const cellData = () => { + const count = props.mlStore.outputCount(); + const outputs = props.mlStore.outputs(); + const names = getParamNames(); + const colors = getParamColors(); + const items: Array<{ index: number; value: number; name: string; color: string }> = []; + for (let i = 0; i < count; i++) { + items.push({ + index: i, + value: outputs[i] ?? 0, + name: names[i] ?? `p${i}`, + color: colors[i] ?? defaultParamColor(i), + }); + } + return items; + }; + + return ( +
+
+ + {(item) => { + const barWidth = () => `${Math.max(2, Math.round(item.value * 100))}%`; + const handlers = createCellHandlers(item.index); + return ( +
+
+
+ ); + }} + +
+
+ {tooltipText()} +
+ + {(p) => { + const outputs = props.mlStore.outputs(); + const value = outputs[p().paramIndex] ?? 0; + const names = getParamNames(); + const name = names[p().paramIndex] ?? `p${p().paramIndex}`; + const colors = getParamColors(); + const color = colors[p().paramIndex] ?? defaultParamColor(p().paramIndex); + const barWidth = `${Math.max(2, Math.round(value * 100))}%`; + + return ( +
+
+ {name} + {value.toFixed(3)} + +
+
+
+
+
+ ); + }} + +
+ ); +} diff --git a/src/components/visual/heatmap-strip.css b/src/components/visual/heatmap-strip.css new file mode 100644 index 0000000..eb04bf8 --- /dev/null +++ b/src/components/visual/heatmap-strip.css @@ -0,0 +1,134 @@ +/* ---- Heatmap strip (top) — interactive bars ---- */ +.heatmap-strip { + position: fixed; + top: 0; + left: 0; + right: 0; + height: 22px; + z-index: 20; + background: rgba(13, 13, 13, 0.5); + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); + display: flex; + align-items: flex-end; + padding: 2px 4px; +} + +.heatmap-cells { + display: flex; + flex-direction: row; + flex: 1; + height: 16px; + gap: 1px; + border-radius: 3px; + overflow: hidden; + align-items: stretch; +} + +.heatmap-cell { + flex: 1; + min-width: 0; + position: relative; + cursor: pointer; + height: 16px; + background: rgba(255, 255, 255, 0.04); + overflow: hidden; +} + +.heatmap-cell.dragging { + cursor: ew-resize; +} + +.heatmap-cell-bar { + height: 100%; + border-radius: 0 1px 1px 0; + transition: width 0.15s; + pointer-events: none; +} + +.heatmap-cell:hover .heatmap-cell-bar { + filter: brightness(1.3); +} + +.heatmap-tooltip { + position: fixed; + top: 32px; + padding: 4px 10px; + background: rgba(0, 0, 0, 0.85); + border: 1px solid var(--glass-border, rgba(255, 255, 255, 0.12)); + border-radius: 6px; + font-size: 12px; + color: var(--text, #e0e0e8); + pointer-events: none; + opacity: 0; + transition: opacity 0.15s; + z-index: 25; + white-space: nowrap; +} + +.heatmap-tooltip.visible { + opacity: 1; +} + +/* Param popup */ +.param-popup { + position: fixed; + z-index: 30; + background: rgba(13, 13, 20, 0.92); + border: 1px solid var(--glass-border, rgba(255, 255, 255, 0.12)); + border-radius: 8px; + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + padding: 10px 12px; + min-width: 200px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.5); +} + +.param-popup-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.param-popup-name { + font-size: 12px; + font-weight: 600; + color: #fff; +} + +.param-popup-value { + font-size: 11px; + color: var(--accent, #00d4ff); + font-variant-numeric: tabular-nums; +} + +.param-popup-close { + background: none; + border: none; + color: #888; + font-size: 16px; + cursor: pointer; + padding: 0 2px; + line-height: 1; +} + +.param-popup-close:hover { + color: #fff; +} + +.param-popup-bar { + width: 100%; + height: 6px; + background: rgba(255, 255, 255, 0.08); + border-radius: 3px; + margin-top: 6px; + position: relative; + overflow: hidden; +} + +.param-popup-bar-fill { + height: 100%; + border-radius: 3px; + transition: width 0.15s; +} diff --git a/src/stores/ml-store.ts b/src/stores/ml-store.ts index a6c4ad4..6fc01f8 100644 --- a/src/stores/ml-store.ts +++ b/src/stores/ml-store.ts @@ -158,6 +158,9 @@ export interface MLStore { loadVisualPreset(name: string): void; saveState(): void; + /** Internal: update outputs signal from external source (e.g. heatmap drag) */ + _updateOutputs(newOutputs: Float32Array): void; + // Lifecycle dispose(): void; } @@ -235,6 +238,13 @@ export async function createMLStore(bus: SignalBus): Promise { setLastLossSignal(activeIml.lastLoss); } + // ─── Helper: update outputs signal from external source (e.g. heatmap drag) ─── + + function _updateOutputs(newOutputs: Float32Array): void { + setOutputs(newOutputs); + outputsTopic.emit(newOutputs); + } + // ─── Helper: push undo snapshot ─── function pushUndoSnapshot(): void { @@ -513,6 +523,10 @@ export async function createMLStore(bus: SignalBus): Promise { localStorage.setItem('nisps-a-immersive', JSON.stringify(data)); }, + _updateOutputs: (newOutputs: Float32Array) => { + _updateOutputs(newOutputs); + }, + dispose: () => { imlJoy?.destroy(); imlHand?.destroy(); diff --git a/tests/solidjs/heatmap-strip.spec.ts b/tests/solidjs/heatmap-strip.spec.ts new file mode 100644 index 0000000..832a6cd --- /dev/null +++ b/tests/solidjs/heatmap-strip.spec.ts @@ -0,0 +1,314 @@ +/** + * e2e tests for f14-heatmap-strip. + * Validates: + * VAL-HEAT-001: Bar count matches mode + * VAL-HEAT-002: Bar widths reflect outputs + * VAL-HEAT-003: Bars update on inference + * VAL-HEAT-004: Drag on bar sets value + * VAL-HEAT-005: Click opens param popup + * VAL-HEAT-006: Heatmap rebuilds on mode switch + */ +import { test, expect } from '@playwright/test'; +import { loadSolidApp } from './helpers'; + +test.describe('Heatmap Strip', () => { + + // ─── VAL-HEAT-001: Bar count matches mode ─── + + test('VAL-HEAT-001a: visual mode shows 20 heatmap bars', async ({ page }) => { + await loadSolidApp(page); + + const barCount = await page.evaluate(() => { + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + return cells.length; + }); + + expect(barCount).toBe(20); + }); + + test('VAL-HEAT-001b: synth mode shows 126 heatmap bars', async ({ page }) => { + await loadSolidApp(page); + + const barCount = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + await store.setOutputMode('synth'); + // Wait for DOM update + await new Promise(r => setTimeout(r, 100)); + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + return cells.length; + }); + + expect(barCount).toBe(126); + }); + + test('VAL-HEAT-001c: midi-cc mode shows 8 heatmap bars', async ({ page }) => { + await loadSolidApp(page); + + const barCount = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + await store.setOutputMode('midi-cc'); + await new Promise(r => setTimeout(r, 100)); + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + return cells.length; + }); + + expect(barCount).toBe(8); + }); + + test('VAL-HEAT-001d: audio-canvas mode shows 36 heatmap bars', async ({ page }) => { + await loadSolidApp(page); + + const barCount = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + await store.setOutputMode('audio-canvas'); + await new Promise(r => setTimeout(r, 100)); + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + return cells.length; + }); + + expect(barCount).toBe(36); + }); + + // ─── VAL-HEAT-002: Bar widths reflect outputs ─── + + test('VAL-HEAT-002a: each heatmap bar has width proportional to output value', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const outputs = probe.getOutputs(); + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + + // Check first 5 bars + const checks: Array<{ index: number; output: number; barWidthPct: number }> = []; + for (let i = 0; i < Math.min(5, cells.length); i++) { + const bar = cells[i].querySelector('.heatmap-cell-bar') as HTMLElement; + if (!bar) continue; + const widthStr = bar.style.width; + const barWidthPct = parseFloat(widthStr); + checks.push({ index: i, output: outputs[i], barWidthPct }); + } + return checks; + }); + + // Each bar width should be roughly proportional to the output value + for (const check of result) { + // Allow 5% tolerance (minimum bar width is 2%) + const expectedMin = Math.max(2, Math.round(check.output * 100) - 5); + const expectedMax = Math.round(check.output * 100) + 5; + expect(check.barWidthPct).toBeGreaterThanOrEqual(expectedMin); + expect(check.barWidthPct).toBeLessThanOrEqual(expectedMax); + } + }); + + // ─── VAL-HEAT-003: Bars update on inference ─── + + test('VAL-HEAT-003a: bars update when setInputs changes outputs', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + + // Get initial bar widths + const getBarWidths = () => { + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + return Array.from(cells).slice(0, 5).map(cell => { + const bar = cell.querySelector('.heatmap-cell-bar') as HTMLElement; + return bar ? parseFloat(bar.style.width) : 0; + }); + }; + + const widthsBefore = getBarWidths(); + + // Change inputs to a different position + probe.setInputs(0.9, 0.1); + + const widthsAfter = getBarWidths(); + + return { widthsBefore, widthsAfter }; + }); + + // At least one bar should have changed width + let anyChanged = false; + for (let i = 0; i < result.widthsBefore.length; i++) { + if (Math.abs(result.widthsBefore[i] - result.widthsAfter[i]) > 0.5) { + anyChanged = true; + break; + } + } + expect(anyChanged).toBe(true); + }); + + test('VAL-HEAT-003b: bar widths match output values after inference', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + probe.setInputs(0.3, 0.7); + + const outputs = probe.getOutputs(); + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + + // Check a few bars + const checks: Array<{ output: number; barWidthPct: number }> = []; + for (let i = 0; i < Math.min(5, cells.length); i++) { + const bar = cells[i].querySelector('.heatmap-cell-bar') as HTMLElement; + if (!bar) continue; + const barWidthPct = parseFloat(bar.style.width); + checks.push({ output: outputs[i], barWidthPct }); + } + return checks; + }); + + for (const check of result) { + const expectedPct = Math.max(2, Math.round(check.output * 100)); + expect(Math.abs(check.barWidthPct - expectedPct)).toBeLessThanOrEqual(5); + } + }); + + // ─── VAL-HEAT-004: Drag on bar sets value ─── + + test('VAL-HEAT-004a: dragging a heatmap bar changes the output value', async ({ page }) => { + await loadSolidApp(page); + + // Locate first heatmap cell + const firstCell = page.locator('#heatmap-cells .heatmap-cell').first(); + await expect(firstCell).toBeVisible(); + + // Use programmatic drag: set value directly via pointer events + const result = await page.evaluate(async () => { + const cell = document.querySelector('#heatmap-cells .heatmap-cell') as HTMLElement; + if (!cell) return { success: false, error: 'no cell' }; + + const rect = cell.getBoundingClientRect(); + + // Simulate a drag from left to center of cell + // pointerdown near left edge + cell.dispatchEvent(new PointerEvent('pointerdown', { + clientX: rect.left + 2, + clientY: rect.top + rect.height / 2, + pointerId: 1, + bubbles: true, + cancelable: true, + })); + + // pointermove to center + cell.dispatchEvent(new PointerEvent('pointermove', { + clientX: rect.left + rect.width / 2, + clientY: rect.top + rect.height / 2, + pointerId: 1, + bubbles: true, + cancelable: true, + })); + + // pointerup at center + cell.dispatchEvent(new PointerEvent('pointerup', { + clientX: rect.left + rect.width / 2, + clientY: rect.top + rect.height / 2, + pointerId: 1, + bubbles: true, + cancelable: true, + })); + + // Wait a tick for reactive update + await new Promise(r => setTimeout(r, 50)); + + const bar = cell.querySelector('.heatmap-cell-bar') as HTMLElement; + const barWidth = bar ? parseFloat(bar.style.width) : -1; + + return { success: true, barWidth, cellWidth: rect.width }; + }); + + expect(result.success).toBe(true); + // Should be close to 50% (within 25% tolerance for drag imprecision) + expect(result.barWidth).toBeGreaterThan(25); + expect(result.barWidth).toBeLessThan(75); + }); + + // ─── VAL-HEAT-005: Click opens param popup ─── + + test('VAL-HEAT-005a: clicking a heatmap cell opens param popup', async ({ page }) => { + await loadSolidApp(page); + + // Click the first heatmap cell (short click, not drag) + const firstCell = page.locator('#heatmap-cells .heatmap-cell').first(); + await expect(firstCell).toBeVisible(); + + const box = await firstCell.boundingBox(); + expect(box).not.toBeNull(); + + // Short click in center of cell + await page.mouse.click(box!.x + box!.width / 2, box!.y + box!.height / 2); + + // Check that param popup is visible + const popup = page.locator('#param-popup'); + await expect(popup).toBeVisible({ timeout: 3000 }); + }); + + // ─── VAL-HEAT-006: Heatmap rebuilds on mode switch ─── + + test('VAL-HEAT-006a: heatmap rebuilds with correct count after mode switch', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + + // Check visual mode (20 bars) + const visualCells = document.querySelectorAll('#heatmap-cells .heatmap-cell').length; + + // Switch to synth + await store.setOutputMode('synth'); + await new Promise(r => setTimeout(r, 100)); + const synthCells = document.querySelectorAll('#heatmap-cells .heatmap-cell').length; + + // Switch back to visual + await store.setOutputMode('visual'); + await new Promise(r => setTimeout(r, 100)); + const visualCellsAgain = document.querySelectorAll('#heatmap-cells .heatmap-cell').length; + + return { visualCells, synthCells, visualCellsAgain }; + }); + + expect(result.visualCells).toBe(20); + expect(result.synthCells).toBe(126); + expect(result.visualCellsAgain).toBe(20); + }); + + test('VAL-HEAT-006b: heatmap bars show correct colors after rebuild', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const cells = document.querySelectorAll('#heatmap-cells .heatmap-cell'); + // Visual mode bars should have colored backgrounds + const firstBar = cells[0]?.querySelector('.heatmap-cell-bar') as HTMLElement; + return { + hasBar: !!firstBar, + hasBackground: firstBar ? firstBar.style.background !== '' : false, + barCount: cells.length, + }; + }); + + expect(result.hasBar).toBe(true); + expect(result.hasBackground).toBe(true); + expect(result.barCount).toBe(20); + }); + + test('VAL-HEAT-006c: tooltip shows on hover over heatmap cell', async ({ page }) => { + await loadSolidApp(page); + + const firstCell = page.locator('#heatmap-cells .heatmap-cell').first(); + await expect(firstCell).toBeVisible(); + + // Hover over first cell + await firstCell.hover(); + + // Tooltip should become visible + const tooltip = page.locator('#heatmap-tooltip'); + await expect(tooltip).toBeVisible({ timeout: 3000 }); + + // Tooltip should contain parameter name and value + const text = await tooltip.textContent(); + expect(text).toBeTruthy(); + expect(text!.length).toBeGreaterThan(3); + }); +});