501 lines
15 KiB
TypeScript
501 lines
15 KiB
TypeScript
|
|
/**
|
||
|
|
* e2e tests for f11-training-sync-async-loss.
|
||
|
|
* Validates:
|
||
|
|
* VAL-ML-006: Add example captures current state (thumbsUp variant)
|
||
|
|
* VAL-ML-007: Sync training returns finite loss
|
||
|
|
* VAL-ML-008: Training converges (second loss <= first)
|
||
|
|
* VAL-ML-009: Async training via worker returns Promise resolving to finite loss
|
||
|
|
* VAL-ML-010: Loss history populated with multiple entries after training
|
||
|
|
*/
|
||
|
|
import { test, expect } from '@playwright/test';
|
||
|
|
import { loadSolidApp } from './helpers';
|
||
|
|
|
||
|
|
test.describe('Training: Sync, Async, Loss History (f11)', () => {
|
||
|
|
|
||
|
|
// ─── VAL-ML-006: Add example captures current state (thumbsUp) ───
|
||
|
|
|
||
|
|
test('VAL-ML-006a: thumbsUp increments exampleCount by 1', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
const countBefore = probe.getExampleCount();
|
||
|
|
await probe.thumbsUp();
|
||
|
|
const countAfter = probe.getExampleCount();
|
||
|
|
|
||
|
|
return { countBefore, countAfter };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.countBefore).toBe(0);
|
||
|
|
expect(result.countAfter).toBe(1);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-006b: thumbsUp captures feature with correct dimensions', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Set a known input position
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
|
||
|
|
// ThumbsUp captures current state
|
||
|
|
await probe.thumbsUp();
|
||
|
|
|
||
|
|
// Inspect the captured example via the active IML
|
||
|
|
const iml = store.getActiveIml();
|
||
|
|
const lastFeature = iml.dataset.features[iml.dataset.features.length - 1];
|
||
|
|
const lastLabel = iml.dataset.labels[iml.dataset.labels.length - 1];
|
||
|
|
|
||
|
|
return {
|
||
|
|
featureLength: lastFeature?.length,
|
||
|
|
labelLength: lastLabel?.length,
|
||
|
|
featureBounded: lastFeature?.every((v: number) => v >= 0 && v <= 1),
|
||
|
|
labelBounded: lastLabel?.every((v: number) => v >= 0 && v <= 1),
|
||
|
|
featureValues: lastFeature?.slice(0, 2), // first 2 should be 0.3, 0.7
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
// Feature is [x, y] = 2 values (without bias — bias is added internally)
|
||
|
|
expect(result.featureLength).toBe(2);
|
||
|
|
// Label matches current output count (20 in visual mode)
|
||
|
|
expect(result.labelLength).toBe(20);
|
||
|
|
expect(result.featureBounded).toBe(true);
|
||
|
|
expect(result.labelBounded).toBe(true);
|
||
|
|
// First 2 features should match our setInputs values
|
||
|
|
expect(result.featureValues![0]).toBeCloseTo(0.3, 1);
|
||
|
|
expect(result.featureValues![1]).toBeCloseTo(0.7, 1);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-006c: multiple thumbsUp calls increment count sequentially', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const counts: number[] = [probe.getExampleCount()];
|
||
|
|
|
||
|
|
// Move to different positions and thumbs-up each
|
||
|
|
probe.setInputs(0.2, 0.8);
|
||
|
|
await probe.thumbsUp();
|
||
|
|
counts.push(probe.getExampleCount());
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
await probe.thumbsUp();
|
||
|
|
counts.push(probe.getExampleCount());
|
||
|
|
|
||
|
|
probe.setInputs(0.8, 0.2);
|
||
|
|
await probe.thumbsUp();
|
||
|
|
counts.push(probe.getExampleCount());
|
||
|
|
|
||
|
|
return { counts };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.counts).toEqual([0, 1, 2, 3]);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── VAL-ML-007: Sync training returns finite loss ───
|
||
|
|
|
||
|
|
test('VAL-ML-007a: train() returns null with no examples', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
return probe.train();
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result).toBeNull();
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-007b: train() returns finite non-negative loss with examples', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
// Add examples at different positions
|
||
|
|
probe.setInputs(0.2, 0.8);
|
||
|
|
probe.thumbsUp(); // sync addExample via store — not probe.thumbsUp which is async
|
||
|
|
// Wait — probe.thumbsUp is async, but we need just addExample.
|
||
|
|
// Actually probe.thumbsUp adds example AND trains async.
|
||
|
|
// Use store.addExample directly for adding without training.
|
||
|
|
return null;
|
||
|
|
});
|
||
|
|
|
||
|
|
// Better approach: add examples manually then train sync
|
||
|
|
const loss = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add examples at different positions
|
||
|
|
probe.setInputs(0.2, 0.8);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.8, 0.2);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
// Train synchronously
|
||
|
|
const loss = probe.train();
|
||
|
|
return loss;
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(loss).not.toBeNull();
|
||
|
|
expect(typeof loss).toBe('number');
|
||
|
|
expect(Number.isFinite(loss as number)).toBe(true);
|
||
|
|
expect(loss as number).toBeGreaterThanOrEqual(0);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-007c: train() returns a number (not NaN)', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
probe.setInputs(0.4, 0.6);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.6, 0.4);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
const loss = probe.train();
|
||
|
|
return {
|
||
|
|
loss,
|
||
|
|
isNumber: typeof loss === 'number',
|
||
|
|
isNotNaN: loss !== null && !Number.isNaN(loss),
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.isNumber).toBe(true);
|
||
|
|
expect(result.isNotNaN).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── VAL-ML-008: Training converges ───
|
||
|
|
|
||
|
|
test('VAL-ML-008a: second train() loss <= first train() loss', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add several examples
|
||
|
|
probe.setInputs(0.1, 0.9);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.7, 0.3);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.9, 0.1);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
// First training
|
||
|
|
const loss1 = probe.train();
|
||
|
|
|
||
|
|
// Second training (same examples, should converge or stay same)
|
||
|
|
const loss2 = probe.train();
|
||
|
|
|
||
|
|
return { loss1, loss2 };
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.loss1).not.toBeNull();
|
||
|
|
expect(result.loss2).not.toBeNull();
|
||
|
|
// Second loss should be <= first (within epsilon)
|
||
|
|
expect(result.loss2!).toBeLessThanOrEqual(result.loss1! + 1e-6);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-008b: repeated training continues to converge', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add examples
|
||
|
|
probe.setInputs(0.2, 0.8);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.8, 0.2);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
// Train multiple times and collect losses
|
||
|
|
const losses: (number | null)[] = [];
|
||
|
|
for (let i = 0; i < 5; i++) {
|
||
|
|
losses.push(probe.train());
|
||
|
|
}
|
||
|
|
|
||
|
|
return { losses };
|
||
|
|
});
|
||
|
|
|
||
|
|
// All losses should be finite non-negative
|
||
|
|
for (const loss of result.losses) {
|
||
|
|
expect(loss).not.toBeNull();
|
||
|
|
expect(Number.isFinite(loss as number)).toBe(true);
|
||
|
|
expect(loss as number).toBeGreaterThanOrEqual(0);
|
||
|
|
}
|
||
|
|
|
||
|
|
// General trend: later losses <= earlier ones (allow small epsilon for numerical noise)
|
||
|
|
const first = result.losses[0] as number;
|
||
|
|
const last = result.losses[result.losses.length - 1] as number;
|
||
|
|
expect(last).toBeLessThanOrEqual(first + 1e-4);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── VAL-ML-009: Async training via worker ───
|
||
|
|
|
||
|
|
test('VAL-ML-009a: trainAsync() returns a Promise', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const isPromise = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add an example first
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
const result = probe.trainAsync();
|
||
|
|
return result instanceof Promise;
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(isPromise).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-009b: trainAsync() resolves to finite loss', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const loss = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add examples
|
||
|
|
probe.setInputs(0.2, 0.8);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.5, 0.5);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.8, 0.2);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
const loss = await probe.trainAsync();
|
||
|
|
return loss;
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(loss).not.toBeNull();
|
||
|
|
expect(typeof loss).toBe('number');
|
||
|
|
expect(Number.isFinite(loss as number)).toBe(true);
|
||
|
|
expect(loss as number).toBeGreaterThanOrEqual(0);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-009c: trainAsync() returns null with no examples', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const loss = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
return await probe.trainAsync();
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(loss).toBeNull();
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-009d: trainAsync() updates outputs after completion', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Capture outputs before training at a specific position
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
const outputsAtInput = probe.getOutputs().slice();
|
||
|
|
|
||
|
|
// Add an example at a very different position with target outputs
|
||
|
|
// that differ from current behavior — this ensures training will change
|
||
|
|
// the outputs at our test position
|
||
|
|
store.addExample(); // captures at (0.3, 0.7)
|
||
|
|
probe.setInputs(0.8, 0.2);
|
||
|
|
store.addExample(); // captures at (0.8, 0.2)
|
||
|
|
|
||
|
|
// Get outputs at (0.8, 0.2) before training
|
||
|
|
const beforeTraining = probe.getOutputs().slice();
|
||
|
|
|
||
|
|
const loss = await probe.trainAsync();
|
||
|
|
|
||
|
|
// After training, outputs at the same input position may have changed
|
||
|
|
const afterTraining = probe.getOutputs().slice();
|
||
|
|
|
||
|
|
return {
|
||
|
|
loss,
|
||
|
|
lossIsFinite: loss !== null && Number.isFinite(loss),
|
||
|
|
// Verify trainAsync completed and updated the IML state
|
||
|
|
outputsAfterTraining: afterTraining.length,
|
||
|
|
allBounded: afterTraining.every((v: number) => v >= 0 && v <= 1),
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.loss).not.toBeNull();
|
||
|
|
expect(result.lossIsFinite).toBe(true);
|
||
|
|
// After async training, outputs should still be valid
|
||
|
|
expect(result.outputsAfterTraining).toBeGreaterThan(0);
|
||
|
|
expect(result.allBounded).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── VAL-ML-010: Loss history populated after training ───
|
||
|
|
|
||
|
|
test('VAL-ML-010a: lossHistory has multiple entries after sync training', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
// Add examples
|
||
|
|
probe.setInputs(0.2, 0.8);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.8, 0.2);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
// Train twice to generate more loss history
|
||
|
|
probe.train();
|
||
|
|
probe.train();
|
||
|
|
|
||
|
|
const history = probe.getLossHistory();
|
||
|
|
return {
|
||
|
|
length: history.length,
|
||
|
|
allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0),
|
||
|
|
firstFew: history.slice(0, 5),
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
// Loss history should have multiple entries (train_ex captures per-iteration losses)
|
||
|
|
expect(result.length).toBeGreaterThan(1);
|
||
|
|
expect(result.allFinite).toBe(true);
|
||
|
|
// All values should be non-negative
|
||
|
|
for (const v of result.firstFew) {
|
||
|
|
expect(v).toBeGreaterThanOrEqual(0);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-010b: lossHistory grows with each training call', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(() => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.7, 0.3);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
const lengths: number[] = [];
|
||
|
|
|
||
|
|
// Before training
|
||
|
|
lengths.push(probe.getLossHistory().length);
|
||
|
|
|
||
|
|
// After first training
|
||
|
|
probe.train();
|
||
|
|
lengths.push(probe.getLossHistory().length);
|
||
|
|
|
||
|
|
// After second training
|
||
|
|
probe.train();
|
||
|
|
lengths.push(probe.getLossHistory().length);
|
||
|
|
|
||
|
|
return { lengths };
|
||
|
|
});
|
||
|
|
|
||
|
|
// History should grow (or stay same) with each training call
|
||
|
|
expect(result.lengths[1]).toBeGreaterThanOrEqual(result.lengths[0]);
|
||
|
|
expect(result.lengths[2]).toBeGreaterThanOrEqual(result.lengths[1]);
|
||
|
|
// At least one entry should be added by training
|
||
|
|
expect(result.lengths[2]).toBeGreaterThan(0);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('VAL-ML-010c: lossHistory populated after async training', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
const store = (window as any).__nispsStore;
|
||
|
|
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
probe.setInputs(0.7, 0.3);
|
||
|
|
store.addExample();
|
||
|
|
|
||
|
|
// Async train
|
||
|
|
await probe.trainAsync();
|
||
|
|
|
||
|
|
const history = probe.getLossHistory();
|
||
|
|
return {
|
||
|
|
length: history.length,
|
||
|
|
allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0),
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.length).toBeGreaterThan(0);
|
||
|
|
expect(result.allFinite).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
// ─── Cross-checks: thumbsUp triggers training ───
|
||
|
|
|
||
|
|
test('thumbsUp adds example AND triggers async training (loss updates)', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
// Set initial position
|
||
|
|
probe.setInputs(0.3, 0.7);
|
||
|
|
|
||
|
|
// ThumbsUp should add example + train async
|
||
|
|
const loss = await probe.thumbsUp();
|
||
|
|
|
||
|
|
return {
|
||
|
|
exampleCount: probe.getExampleCount(),
|
||
|
|
loss,
|
||
|
|
lossIsFinite: loss !== null && Number.isFinite(loss),
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.exampleCount).toBeGreaterThanOrEqual(1);
|
||
|
|
expect(result.lossIsFinite).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('after thumbsUp, lossHistory has entries', async ({ page }) => {
|
||
|
|
await loadSolidApp(page);
|
||
|
|
|
||
|
|
const result = await page.evaluate(async () => {
|
||
|
|
const probe = (window as any).__nisps;
|
||
|
|
|
||
|
|
probe.setInputs(0.4, 0.6);
|
||
|
|
await probe.thumbsUp();
|
||
|
|
|
||
|
|
probe.setInputs(0.6, 0.4);
|
||
|
|
await probe.thumbsUp();
|
||
|
|
|
||
|
|
const history = probe.getLossHistory();
|
||
|
|
return {
|
||
|
|
length: history.length,
|
||
|
|
allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0),
|
||
|
|
};
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(result.length).toBeGreaterThan(0);
|
||
|
|
expect(result.allFinite).toBe(true);
|
||
|
|
});
|
||
|
|
});
|