feat: add training sync/async/loss tests validating VAL-ML-006 through VAL-ML-010 (f11)
17 e2e tests covering thumbsUp example capture, sync train returning finite loss, training convergence, async train via worker resolving to finite loss, and loss history population. All existing and new tests pass.
This commit is contained in:
parent
fad057a108
commit
b90a7177fd
1 changed files with 500 additions and 0 deletions
500
tests/solidjs/training-sync-async-loss.spec.ts
Normal file
500
tests/solidjs/training-sync-async-loss.spec.ts
Normal file
|
|
@ -0,0 +1,500 @@
|
|||
/**
|
||||
* e2e tests for f11-training-sync-async-loss.
|
||||
* Validates:
|
||||
* VAL-ML-006: Add example captures current state (thumbsUp variant)
|
||||
* VAL-ML-007: Sync training returns finite loss
|
||||
* VAL-ML-008: Training converges (second loss <= first)
|
||||
* VAL-ML-009: Async training via worker returns Promise resolving to finite loss
|
||||
* VAL-ML-010: Loss history populated with multiple entries after training
|
||||
*/
|
||||
import { test, expect } from '@playwright/test';
|
||||
import { loadSolidApp } from './helpers';
|
||||
|
||||
test.describe('Training: Sync, Async, Loss History (f11)', () => {
|
||||
|
||||
// ─── VAL-ML-006: Add example captures current state (thumbsUp) ───
|
||||
|
||||
test('VAL-ML-006a: thumbsUp increments exampleCount by 1', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
|
||||
const countBefore = probe.getExampleCount();
|
||||
await probe.thumbsUp();
|
||||
const countAfter = probe.getExampleCount();
|
||||
|
||||
return { countBefore, countAfter };
|
||||
});
|
||||
|
||||
expect(result.countBefore).toBe(0);
|
||||
expect(result.countAfter).toBe(1);
|
||||
});
|
||||
|
||||
test('VAL-ML-006b: thumbsUp captures feature with correct dimensions', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Set a known input position
|
||||
probe.setInputs(0.3, 0.7);
|
||||
|
||||
// ThumbsUp captures current state
|
||||
await probe.thumbsUp();
|
||||
|
||||
// Inspect the captured example via the active IML
|
||||
const iml = store.getActiveIml();
|
||||
const lastFeature = iml.dataset.features[iml.dataset.features.length - 1];
|
||||
const lastLabel = iml.dataset.labels[iml.dataset.labels.length - 1];
|
||||
|
||||
return {
|
||||
featureLength: lastFeature?.length,
|
||||
labelLength: lastLabel?.length,
|
||||
featureBounded: lastFeature?.every((v: number) => v >= 0 && v <= 1),
|
||||
labelBounded: lastLabel?.every((v: number) => v >= 0 && v <= 1),
|
||||
featureValues: lastFeature?.slice(0, 2), // first 2 should be 0.3, 0.7
|
||||
};
|
||||
});
|
||||
|
||||
// Feature is [x, y] = 2 values (without bias — bias is added internally)
|
||||
expect(result.featureLength).toBe(2);
|
||||
// Label matches current output count (20 in visual mode)
|
||||
expect(result.labelLength).toBe(20);
|
||||
expect(result.featureBounded).toBe(true);
|
||||
expect(result.labelBounded).toBe(true);
|
||||
// First 2 features should match our setInputs values
|
||||
expect(result.featureValues![0]).toBeCloseTo(0.3, 1);
|
||||
expect(result.featureValues![1]).toBeCloseTo(0.7, 1);
|
||||
});
|
||||
|
||||
test('VAL-ML-006c: multiple thumbsUp calls increment count sequentially', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
const counts: number[] = [probe.getExampleCount()];
|
||||
|
||||
// Move to different positions and thumbs-up each
|
||||
probe.setInputs(0.2, 0.8);
|
||||
await probe.thumbsUp();
|
||||
counts.push(probe.getExampleCount());
|
||||
|
||||
probe.setInputs(0.5, 0.5);
|
||||
await probe.thumbsUp();
|
||||
counts.push(probe.getExampleCount());
|
||||
|
||||
probe.setInputs(0.8, 0.2);
|
||||
await probe.thumbsUp();
|
||||
counts.push(probe.getExampleCount());
|
||||
|
||||
return { counts };
|
||||
});
|
||||
|
||||
expect(result.counts).toEqual([0, 1, 2, 3]);
|
||||
});
|
||||
|
||||
// ─── VAL-ML-007: Sync training returns finite loss ───
|
||||
|
||||
test('VAL-ML-007a: train() returns null with no examples', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
return probe.train();
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('VAL-ML-007b: train() returns finite non-negative loss with examples', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
|
||||
// Add examples at different positions
|
||||
probe.setInputs(0.2, 0.8);
|
||||
probe.thumbsUp(); // sync addExample via store — not probe.thumbsUp which is async
|
||||
// Wait — probe.thumbsUp is async, but we need just addExample.
|
||||
// Actually probe.thumbsUp adds example AND trains async.
|
||||
// Use store.addExample directly for adding without training.
|
||||
return null;
|
||||
});
|
||||
|
||||
// Better approach: add examples manually then train sync
|
||||
const loss = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Add examples at different positions
|
||||
probe.setInputs(0.2, 0.8);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.5, 0.5);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.8, 0.2);
|
||||
store.addExample();
|
||||
|
||||
// Train synchronously
|
||||
const loss = probe.train();
|
||||
return loss;
|
||||
});
|
||||
|
||||
expect(loss).not.toBeNull();
|
||||
expect(typeof loss).toBe('number');
|
||||
expect(Number.isFinite(loss as number)).toBe(true);
|
||||
expect(loss as number).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('VAL-ML-007c: train() returns a number (not NaN)', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
probe.setInputs(0.4, 0.6);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.6, 0.4);
|
||||
store.addExample();
|
||||
|
||||
const loss = probe.train();
|
||||
return {
|
||||
loss,
|
||||
isNumber: typeof loss === 'number',
|
||||
isNotNaN: loss !== null && !Number.isNaN(loss),
|
||||
};
|
||||
});
|
||||
|
||||
expect(result.isNumber).toBe(true);
|
||||
expect(result.isNotNaN).toBe(true);
|
||||
});
|
||||
|
||||
// ─── VAL-ML-008: Training converges ───
|
||||
|
||||
test('VAL-ML-008a: second train() loss <= first train() loss', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Add several examples
|
||||
probe.setInputs(0.1, 0.9);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.3, 0.7);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.5, 0.5);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.7, 0.3);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.9, 0.1);
|
||||
store.addExample();
|
||||
|
||||
// First training
|
||||
const loss1 = probe.train();
|
||||
|
||||
// Second training (same examples, should converge or stay same)
|
||||
const loss2 = probe.train();
|
||||
|
||||
return { loss1, loss2 };
|
||||
});
|
||||
|
||||
expect(result.loss1).not.toBeNull();
|
||||
expect(result.loss2).not.toBeNull();
|
||||
// Second loss should be <= first (within epsilon)
|
||||
expect(result.loss2!).toBeLessThanOrEqual(result.loss1! + 1e-6);
|
||||
});
|
||||
|
||||
test('VAL-ML-008b: repeated training continues to converge', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Add examples
|
||||
probe.setInputs(0.2, 0.8);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.8, 0.2);
|
||||
store.addExample();
|
||||
|
||||
// Train multiple times and collect losses
|
||||
const losses: (number | null)[] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
losses.push(probe.train());
|
||||
}
|
||||
|
||||
return { losses };
|
||||
});
|
||||
|
||||
// All losses should be finite non-negative
|
||||
for (const loss of result.losses) {
|
||||
expect(loss).not.toBeNull();
|
||||
expect(Number.isFinite(loss as number)).toBe(true);
|
||||
expect(loss as number).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
|
||||
// General trend: later losses <= earlier ones (allow small epsilon for numerical noise)
|
||||
const first = result.losses[0] as number;
|
||||
const last = result.losses[result.losses.length - 1] as number;
|
||||
expect(last).toBeLessThanOrEqual(first + 1e-4);
|
||||
});
|
||||
|
||||
// ─── VAL-ML-009: Async training via worker ───
|
||||
|
||||
test('VAL-ML-009a: trainAsync() returns a Promise', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const isPromise = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Add an example first
|
||||
probe.setInputs(0.5, 0.5);
|
||||
store.addExample();
|
||||
|
||||
const result = probe.trainAsync();
|
||||
return result instanceof Promise;
|
||||
});
|
||||
|
||||
expect(isPromise).toBe(true);
|
||||
});
|
||||
|
||||
test('VAL-ML-009b: trainAsync() resolves to finite loss', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const loss = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Add examples
|
||||
probe.setInputs(0.2, 0.8);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.5, 0.5);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.8, 0.2);
|
||||
store.addExample();
|
||||
|
||||
const loss = await probe.trainAsync();
|
||||
return loss;
|
||||
});
|
||||
|
||||
expect(loss).not.toBeNull();
|
||||
expect(typeof loss).toBe('number');
|
||||
expect(Number.isFinite(loss as number)).toBe(true);
|
||||
expect(loss as number).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('VAL-ML-009c: trainAsync() returns null with no examples', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const loss = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
return await probe.trainAsync();
|
||||
});
|
||||
|
||||
expect(loss).toBeNull();
|
||||
});
|
||||
|
||||
test('VAL-ML-009d: trainAsync() updates outputs after completion', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Capture outputs before training at a specific position
|
||||
probe.setInputs(0.3, 0.7);
|
||||
const outputsAtInput = probe.getOutputs().slice();
|
||||
|
||||
// Add an example at a very different position with target outputs
|
||||
// that differ from current behavior — this ensures training will change
|
||||
// the outputs at our test position
|
||||
store.addExample(); // captures at (0.3, 0.7)
|
||||
probe.setInputs(0.8, 0.2);
|
||||
store.addExample(); // captures at (0.8, 0.2)
|
||||
|
||||
// Get outputs at (0.8, 0.2) before training
|
||||
const beforeTraining = probe.getOutputs().slice();
|
||||
|
||||
const loss = await probe.trainAsync();
|
||||
|
||||
// After training, outputs at the same input position may have changed
|
||||
const afterTraining = probe.getOutputs().slice();
|
||||
|
||||
return {
|
||||
loss,
|
||||
lossIsFinite: loss !== null && Number.isFinite(loss),
|
||||
// Verify trainAsync completed and updated the IML state
|
||||
outputsAfterTraining: afterTraining.length,
|
||||
allBounded: afterTraining.every((v: number) => v >= 0 && v <= 1),
|
||||
};
|
||||
});
|
||||
|
||||
expect(result.loss).not.toBeNull();
|
||||
expect(result.lossIsFinite).toBe(true);
|
||||
// After async training, outputs should still be valid
|
||||
expect(result.outputsAfterTraining).toBeGreaterThan(0);
|
||||
expect(result.allBounded).toBe(true);
|
||||
});
|
||||
|
||||
// ─── VAL-ML-010: Loss history populated after training ───
|
||||
|
||||
test('VAL-ML-010a: lossHistory has multiple entries after sync training', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
// Add examples
|
||||
probe.setInputs(0.2, 0.8);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.8, 0.2);
|
||||
store.addExample();
|
||||
|
||||
// Train twice to generate more loss history
|
||||
probe.train();
|
||||
probe.train();
|
||||
|
||||
const history = probe.getLossHistory();
|
||||
return {
|
||||
length: history.length,
|
||||
allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0),
|
||||
firstFew: history.slice(0, 5),
|
||||
};
|
||||
});
|
||||
|
||||
// Loss history should have multiple entries (train_ex captures per-iteration losses)
|
||||
expect(result.length).toBeGreaterThan(1);
|
||||
expect(result.allFinite).toBe(true);
|
||||
// All values should be non-negative
|
||||
for (const v of result.firstFew) {
|
||||
expect(v).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('VAL-ML-010b: lossHistory grows with each training call', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(() => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
probe.setInputs(0.3, 0.7);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.7, 0.3);
|
||||
store.addExample();
|
||||
|
||||
const lengths: number[] = [];
|
||||
|
||||
// Before training
|
||||
lengths.push(probe.getLossHistory().length);
|
||||
|
||||
// After first training
|
||||
probe.train();
|
||||
lengths.push(probe.getLossHistory().length);
|
||||
|
||||
// After second training
|
||||
probe.train();
|
||||
lengths.push(probe.getLossHistory().length);
|
||||
|
||||
return { lengths };
|
||||
});
|
||||
|
||||
// History should grow (or stay same) with each training call
|
||||
expect(result.lengths[1]).toBeGreaterThanOrEqual(result.lengths[0]);
|
||||
expect(result.lengths[2]).toBeGreaterThanOrEqual(result.lengths[1]);
|
||||
// At least one entry should be added by training
|
||||
expect(result.lengths[2]).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('VAL-ML-010c: lossHistory populated after async training', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
const store = (window as any).__nispsStore;
|
||||
|
||||
probe.setInputs(0.3, 0.7);
|
||||
store.addExample();
|
||||
|
||||
probe.setInputs(0.7, 0.3);
|
||||
store.addExample();
|
||||
|
||||
// Async train
|
||||
await probe.trainAsync();
|
||||
|
||||
const history = probe.getLossHistory();
|
||||
return {
|
||||
length: history.length,
|
||||
allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0),
|
||||
};
|
||||
});
|
||||
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
expect(result.allFinite).toBe(true);
|
||||
});
|
||||
|
||||
// ─── Cross-checks: thumbsUp triggers training ───
|
||||
|
||||
test('thumbsUp adds example AND triggers async training (loss updates)', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
|
||||
// Set initial position
|
||||
probe.setInputs(0.3, 0.7);
|
||||
|
||||
// ThumbsUp should add example + train async
|
||||
const loss = await probe.thumbsUp();
|
||||
|
||||
return {
|
||||
exampleCount: probe.getExampleCount(),
|
||||
loss,
|
||||
lossIsFinite: loss !== null && Number.isFinite(loss),
|
||||
};
|
||||
});
|
||||
|
||||
expect(result.exampleCount).toBeGreaterThanOrEqual(1);
|
||||
expect(result.lossIsFinite).toBe(true);
|
||||
});
|
||||
|
||||
test('after thumbsUp, lossHistory has entries', async ({ page }) => {
|
||||
await loadSolidApp(page);
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
const probe = (window as any).__nisps;
|
||||
|
||||
probe.setInputs(0.4, 0.6);
|
||||
await probe.thumbsUp();
|
||||
|
||||
probe.setInputs(0.6, 0.4);
|
||||
await probe.thumbsUp();
|
||||
|
||||
const history = probe.getLossHistory();
|
||||
return {
|
||||
length: history.length,
|
||||
allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0),
|
||||
};
|
||||
});
|
||||
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
expect(result.allFinite).toBe(true);
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue