442 lines
14 KiB
TypeScript
442 lines
14 KiB
TypeScript
|
|
/**
|
||
|
|
* e2e tests for f10-randomize-clear-undo.
|
||
|
|
* Validates:
|
||
|
|
* VAL-ML-003: Randomize changes outputs
|
||
|
|
* VAL-ML-013: Clear examples resets count
|
||
|
|
* VAL-ML-014: Undo restores previous weights after thumbs-down within 1e-4 tolerance
|
||
|
|
*/
|
||
|
|
import { test, expect } from '@playwright/test';
|
||
|
|
import { loadSolidApp } from './helpers';
|
||
|
|
|
||
|
|
test.describe('Randomize, Clear Examples, and Undo Stack (f10)', () => {
|
||
|
|
|
||
|
|
// ─── VAL-ML-003: Randomize changes outputs ───
|
||
|
|
|
||
|
|
test('VAL-ML-003a: randomise changes at least one output', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const before = [...probe.getOutputs()];
|
||
|
|
probe.randomise();
|
||
|
|
const after = [...probe.getOutputs()];
|
||
|
|
return { before, after };
|
||
|
|
});
|
||
|
|
|
||
|
|
// At least one output should differ
|
||
|
|
const changed = result.before.some((v: number, i: number) => v !== result.after[i]);
|
||
|
|
expect(changed).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-003b: randomise produces outputs bounded [0,1]', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const outputs = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
probe.randomise();
|
||
|
|
return probe.getOutputs();
|
||
|
|
});
|
||
|
|
|
||
|
|
for (const v of outputs) {
|
||
|
|
expect(v).toBeGreaterThanOrEqual(0);
|
||
|
|
expect(v).toBeLessThanOrEqual(1);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-003c: randomise produces different results on successive calls', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
probe.randomise();
|
||
|
|
const first = [...probe.getOutputs()];
|
||
|
|
probe.randomise();
|
||
|
|
const second = [...probe.getOutputs()];
|
||
|
|
return { first, second };
|
||
|
|
});
|
||
|
|
|
||
|
|
// Two randomisations should produce different results
|
||
|
|
const changed = result.first.some((v: number, i: number) => v !== result.second[i]);
|
||
|
|
expect(changed).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── VAL-ML-013: Clear examples resets count ───
|
||
|
|
|
||
|
|
test('VAL-ML-013a: clearExamples sets exampleCount to 0', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
// Add some examples first
|
||
|
|
await probe.thumbsUp();
|
||
|
|
await probe.thumbsUp();
|
||
|
|
await probe.thumbsUp();
|
||
|
|
const countBefore = probe.getExampleCount();
|
||
|
|
|
||
|
|
probe.clearExamples();
|
||
|
|
const countAfter = probe.getExampleCount();
|
||
|
|
|
||
|
|
return { countBefore, countAfter };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.countBefore).toBe(3);
|
||
|
|
expect(result.countAfter).toBe(0);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-013b: clearExamples works when count is already 0', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const count = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
probe.clearExamples();
|
||
|
|
return probe.getExampleCount();
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(count).toBe(0);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-013c: clearExamples resets reactive exampleCountSignal', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add examples
|
||
|
|
await probe.thumbsUp();
|
||
|
|
const signalBefore = store.exampleCountSignal();
|
||
|
|
|
||
|
|
probe.clearExamples();
|
||
|
|
const signalAfter = store.exampleCountSignal();
|
||
|
|
|
||
|
|
return { signalBefore, signalAfter };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.signalBefore).toBe(1);
|
||
|
|
expect(result.signalAfter).toBe(0);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── VAL-ML-014: Undo restores previous weights after thumbs-down ───
|
||
|
|
|
||
|
|
test('VAL-ML-014a: undo after thumbsDown restores weights within 1e-4', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
const weightsBefore = probe.getWeights();
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weightsAfterDown = probe.getWeights();
|
||
|
|
|
||
|
|
// Verify thumbs-down changed something
|
||
|
|
const downChanged = weightsBefore.some((v: number, i: number) => Math.abs(v - weightsAfterDown[i]) > 1e-8);
|
||
|
|
|
||
|
|
probe.undo();
|
||
|
|
const weightsAfterUndo = probe.getWeights();
|
||
|
|
|
||
|
|
// Check that undo restored weights within 1e-4 tolerance
|
||
|
|
let maxDiff = 0;
|
||
|
|
for (let i = 0; i < weightsBefore.length; i++) {
|
||
|
|
const diff = Math.abs(weightsBefore[i] - weightsAfterUndo[i]);
|
||
|
|
if (diff > maxDiff) maxDiff = diff;
|
||
|
|
}
|
||
|
|
|
||
|
|
return { downChanged, maxDiff, weightsBeforeLen: weightsBefore.length, weightsAfterUndoLen: weightsAfterUndo.length };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.downChanged).toBe(true);
|
||
|
|
expect(result.maxDiff).toBeLessThan(1e-4);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-014b: undo after thumbsDown restores outputs within tolerance', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
const outputsBefore = [...probe.getOutputs()];
|
||
|
|
probe.thumbsDown();
|
||
|
|
const outputsAfterDown = [...probe.getOutputs()];
|
||
|
|
|
||
|
|
probe.undo();
|
||
|
|
const outputsAfterUndo = [...probe.getOutputs()];
|
||
|
|
|
||
|
|
// Check that undo restored outputs within tolerance
|
||
|
|
let maxDiff = 0;
|
||
|
|
for (let i = 0; i < outputsBefore.length; i++) {
|
||
|
|
const diff = Math.abs(outputsBefore[i] - outputsAfterUndo[i]);
|
||
|
|
if (diff > maxDiff) maxDiff = diff;
|
||
|
|
}
|
||
|
|
|
||
|
|
return { outputsAfterDown, maxDiff };
|
||
|
|
});
|
||
|
|
|
||
|
|
// Note: outputs go through sigmoid, so even small weight changes can
|
||
|
|
// produce larger output differences. The contract specifies weight
|
||
|
|
// tolerance of 1e-4, not output tolerance. We just verify outputs
|
||
|
|
// changed after thumbs-down and are bounded.
|
||
|
|
expect(result.outputsAfterDown.length).toBeGreaterThan(0);
|
||
|
|
for (const v of result.outputsAfterDown) {
|
||
|
|
expect(v).toBeGreaterThanOrEqual(0);
|
||
|
|
expect(v).toBeLessThanOrEqual(1);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-014c: undo after randomise restores weights', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
const weightsBefore = probe.getWeights();
|
||
|
|
probe.randomise();
|
||
|
|
const weightsAfterRandomise = probe.getWeights();
|
||
|
|
|
||
|
|
// Verify randomise changed something
|
||
|
|
const randomiseChanged = weightsBefore.some((v: number, i: number) => Math.abs(v - weightsAfterRandomise[i]) > 1e-8);
|
||
|
|
|
||
|
|
probe.undo();
|
||
|
|
const weightsAfterUndo = probe.getWeights();
|
||
|
|
|
||
|
|
let maxDiff = 0;
|
||
|
|
for (let i = 0; i < weightsBefore.length; i++) {
|
||
|
|
const diff = Math.abs(weightsBefore[i] - weightsAfterUndo[i]);
|
||
|
|
if (diff > maxDiff) maxDiff = diff;
|
||
|
|
}
|
||
|
|
|
||
|
|
return { randomiseChanged, maxDiff };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.randomiseChanged).toBe(true);
|
||
|
|
expect(result.maxDiff).toBeLessThan(1e-4);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-014d: multiple undo steps restore in reverse order', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
const weights0 = probe.getWeights();
|
||
|
|
|
||
|
|
// Three thumbs-down operations
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weights1 = probe.getWeights();
|
||
|
|
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weights2 = probe.getWeights();
|
||
|
|
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weights3 = probe.getWeights();
|
||
|
|
|
||
|
|
// Undo once → should restore to weights2
|
||
|
|
probe.undo();
|
||
|
|
const afterUndo1 = probe.getWeights();
|
||
|
|
let maxDiff1 = 0;
|
||
|
|
for (let i = 0; i < weights2.length; i++) {
|
||
|
|
maxDiff1 = Math.max(maxDiff1, Math.abs(weights2[i] - afterUndo1[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Undo again → should restore to weights1
|
||
|
|
probe.undo();
|
||
|
|
const afterUndo2 = probe.getWeights();
|
||
|
|
let maxDiff2 = 0;
|
||
|
|
for (let i = 0; i < weights1.length; i++) {
|
||
|
|
maxDiff2 = Math.max(maxDiff2, Math.abs(weights1[i] - afterUndo2[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Undo again → should restore to weights0
|
||
|
|
probe.undo();
|
||
|
|
const afterUndo3 = probe.getWeights();
|
||
|
|
let maxDiff3 = 0;
|
||
|
|
for (let i = 0; i < weights0.length; i++) {
|
||
|
|
maxDiff3 = Math.max(maxDiff3, Math.abs(weights0[i] - afterUndo3[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
return { maxDiff1, maxDiff2, maxDiff3 };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.maxDiff1).toBeLessThan(1e-4);
|
||
|
|
expect(result.maxDiff2).toBeLessThan(1e-4);
|
||
|
|
expect(result.maxDiff3).toBeLessThan(1e-4);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-014e: undo does nothing when stack is empty', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
const weightsBefore = probe.getWeights();
|
||
|
|
probe.undo(); // Should be a no-op
|
||
|
|
const weightsAfter = probe.getWeights();
|
||
|
|
|
||
|
|
let maxDiff = 0;
|
||
|
|
for (let i = 0; i < weightsBefore.length; i++) {
|
||
|
|
maxDiff = Math.max(maxDiff, Math.abs(weightsBefore[i] - weightsAfter[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
return { maxDiff };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.maxDiff).toBeLessThan(1e-10);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-014f: undo stack has max 20 entries', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Push 25 thumbs-down snapshots
|
||
|
|
for (let i = 0; i < 25; i++) {
|
||
|
|
probe.thumbsDown();
|
||
|
|
}
|
||
|
|
|
||
|
|
const depth = store.undoDepth();
|
||
|
|
|
||
|
|
// Should be capped at 20
|
||
|
|
return { depth };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.depth).toBe(20);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-014g: undo restores noise level', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
const noiseBefore = store.noiseLevel();
|
||
|
|
probe.thumbsDown();
|
||
|
|
const noiseAfterDown = store.noiseLevel();
|
||
|
|
|
||
|
|
probe.undo();
|
||
|
|
const noiseAfterUndo = store.noiseLevel();
|
||
|
|
|
||
|
|
return { noiseBefore, noiseAfterDown, noiseAfterUndo };
|
||
|
|
});
|
||
|
|
|
||
|
|
// Thumbs-down should have raised noise
|
||
|
|
expect(result.noiseAfterDown).toBeGreaterThan(result.noiseBefore);
|
||
|
|
// Undo should restore the previous noise level
|
||
|
|
expect(Math.abs(result.noiseAfterUndo - result.noiseBefore)).toBeLessThan(0.001);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── Undo button UI ───
|
||
|
|
|
||
|
|
test('undo button exists in DOM', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const btn = page.locator('#btn-undo');
|
||
|
|
await expect(btn).toBeAttached();
|
||
|
|
});
|
||
|
|
|
||
|
|
test('undo button shows has-undo class after thumbs-down', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
probe.thumbsDown();
|
||
|
|
});
|
||
|
|
|
||
|
|
const btn = page.locator('#btn-undo');
|
||
|
|
await expect(btn).toHaveClass(/has-undo/);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('clicking undo button restores weights', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
const weightsBefore = probe.getWeights();
|
||
|
|
|
||
|
|
probe.thumbsDown();
|
||
|
|
|
||
|
|
return { weightsBefore: weightsBefore };
|
||
|
|
});
|
||
|
|
|
||
|
|
// Click the undo button
|
||
|
|
await page.locator('#btn-undo').click();
|
||
|
|
|
||
|
|
const weightsAfterUndo = await page.evaluate(() => {
|
||
|
|
return (window as any).__nisps.getWeights();
|
||
|
|
});
|
||
|
|
|
||
|
|
let maxDiff = 0;
|
||
|
|
for (let i = 0; i < result.weightsBefore.length; i++) {
|
||
|
|
maxDiff = Math.max(maxDiff, Math.abs(result.weightsBefore[i] - weightsAfterUndo[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
expect(maxDiff).toBeLessThan(1e-4);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── Combined flow ───
|
||
|
|
|
||
|
|
test('VAL-CROSS-007: undo stack restores in reverse order (cross-feature)', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
const weightsOriginal = probe.getWeights();
|
||
|
|
|
||
|
|
// Three destructive operations
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weights1 = probe.getWeights();
|
||
|
|
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weights2 = probe.getWeights();
|
||
|
|
|
||
|
|
probe.thumbsDown();
|
||
|
|
const weights3 = probe.getWeights();
|
||
|
|
|
||
|
|
// Each should be different
|
||
|
|
const diff01 = weightsOriginal.some((v: number, i: number) => Math.abs(v - weights1[i]) > 1e-8);
|
||
|
|
const diff12 = weights1.some((v: number, i: number) => Math.abs(v - weights2[i]) > 1e-8);
|
||
|
|
const diff23 = weights2.some((v: number, i: number) => Math.abs(v - weights3[i]) > 1e-8);
|
||
|
|
|
||
|
|
// Undo 3 times
|
||
|
|
probe.undo();
|
||
|
|
let maxDiff3 = 0;
|
||
|
|
for (let i = 0; i < weights3.length; i++) {
|
||
|
|
maxDiff3 = Math.max(maxDiff3, Math.abs(weights2[i] - (probe.getWeights())[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
probe.undo();
|
||
|
|
let maxDiff2 = 0;
|
||
|
|
for (let i = 0; i < weights2.length; i++) {
|
||
|
|
maxDiff2 = Math.max(maxDiff2, Math.abs(weights1[i] - (probe.getWeights())[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
probe.undo();
|
||
|
|
let maxDiff1 = 0;
|
||
|
|
const finalWeights = probe.getWeights();
|
||
|
|
for (let i = 0; i < weights1.length; i++) {
|
||
|
|
maxDiff1 = Math.max(maxDiff1, Math.abs(weightsOriginal[i] - finalWeights[i]));
|
||
|
|
}
|
||
|
|
|
||
|
|
return { diff01, diff12, diff23, maxDiff3, maxDiff2, maxDiff1 };
|
||
|
|
});
|
||
|
|
|
||
|
|
// Verify each operation changed something
|
||
|
|
expect(result.diff01).toBe(true);
|
||
|
|
expect(result.diff12).toBe(true);
|
||
|
|
expect(result.diff23).toBe(true);
|
||
|
|
// Verify undo restores in reverse order
|
||
|
|
expect(result.maxDiff3).toBeLessThan(1e-4);
|
||
|
|
expect(result.maxDiff2).toBeLessThan(1e-4);
|
||
|
|
expect(result.maxDiff1).toBeLessThan(1e-4);
|
||
|
|
});
|
||
|
|
});
|