diff --git a/tests/solidjs/training-sync-async-loss.spec.ts b/tests/solidjs/training-sync-async-loss.spec.ts new file mode 100644 index 0000000..2fdee19 --- /dev/null +++ b/tests/solidjs/training-sync-async-loss.spec.ts @@ -0,0 +1,500 @@ +/** + * e2e tests for f11-training-sync-async-loss. + * Validates: + * VAL-ML-006: Add example captures current state (thumbsUp variant) + * VAL-ML-007: Sync training returns finite loss + * VAL-ML-008: Training converges (second loss <= first) + * VAL-ML-009: Async training via worker returns Promise resolving to finite loss + * VAL-ML-010: Loss history populated with multiple entries after training + */ +import { test, expect } from '@playwright/test'; +import { loadSolidApp } from './helpers'; + +test.describe('Training: Sync, Async, Loss History (f11)', () => { + + // ─── VAL-ML-006: Add example captures current state (thumbsUp) ─── + + test('VAL-ML-006a: thumbsUp increments exampleCount by 1', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + + const countBefore = probe.getExampleCount(); + await probe.thumbsUp(); + const countAfter = probe.getExampleCount(); + + return { countBefore, countAfter }; + }); + + expect(result.countBefore).toBe(0); + expect(result.countAfter).toBe(1); + }); + + test('VAL-ML-006b: thumbsUp captures feature with correct dimensions', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Set a known input position + probe.setInputs(0.3, 0.7); + + // ThumbsUp captures current state + await probe.thumbsUp(); + + // Inspect the captured example via the active IML + const iml = store.getActiveIml(); + const lastFeature = iml.dataset.features[iml.dataset.features.length - 1]; + const lastLabel = iml.dataset.labels[iml.dataset.labels.length - 1]; + + return { + featureLength: lastFeature?.length, + labelLength: lastLabel?.length, + featureBounded: lastFeature?.every((v: number) => v >= 0 && v <= 1), + labelBounded: lastLabel?.every((v: number) => v >= 0 && v <= 1), + featureValues: lastFeature?.slice(0, 2), // first 2 should be 0.3, 0.7 + }; + }); + + // Feature is [x, y] = 2 values (without bias — bias is added internally) + expect(result.featureLength).toBe(2); + // Label matches current output count (20 in visual mode) + expect(result.labelLength).toBe(20); + expect(result.featureBounded).toBe(true); + expect(result.labelBounded).toBe(true); + // First 2 features should match our setInputs values + expect(result.featureValues![0]).toBeCloseTo(0.3, 1); + expect(result.featureValues![1]).toBeCloseTo(0.7, 1); + }); + + test('VAL-ML-006c: multiple thumbsUp calls increment count sequentially', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + const counts: number[] = [probe.getExampleCount()]; + + // Move to different positions and thumbs-up each + probe.setInputs(0.2, 0.8); + await probe.thumbsUp(); + counts.push(probe.getExampleCount()); + + probe.setInputs(0.5, 0.5); + await probe.thumbsUp(); + counts.push(probe.getExampleCount()); + + probe.setInputs(0.8, 0.2); + await probe.thumbsUp(); + counts.push(probe.getExampleCount()); + + return { counts }; + }); + + expect(result.counts).toEqual([0, 1, 2, 3]); + }); + + // ─── VAL-ML-007: Sync training returns finite loss ─── + + test('VAL-ML-007a: train() returns null with no examples', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + return probe.train(); + }); + + expect(result).toBeNull(); + }); + + test('VAL-ML-007b: train() returns finite non-negative loss with examples', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + + // Add examples at different positions + probe.setInputs(0.2, 0.8); + probe.thumbsUp(); // sync addExample via store — not probe.thumbsUp which is async + // Wait — probe.thumbsUp is async, but we need just addExample. + // Actually probe.thumbsUp adds example AND trains async. + // Use store.addExample directly for adding without training. + return null; + }); + + // Better approach: add examples manually then train sync + const loss = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Add examples at different positions + probe.setInputs(0.2, 0.8); + store.addExample(); + + probe.setInputs(0.5, 0.5); + store.addExample(); + + probe.setInputs(0.8, 0.2); + store.addExample(); + + // Train synchronously + const loss = probe.train(); + return loss; + }); + + expect(loss).not.toBeNull(); + expect(typeof loss).toBe('number'); + expect(Number.isFinite(loss as number)).toBe(true); + expect(loss as number).toBeGreaterThanOrEqual(0); + }); + + test('VAL-ML-007c: train() returns a number (not NaN)', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + probe.setInputs(0.4, 0.6); + store.addExample(); + + probe.setInputs(0.6, 0.4); + store.addExample(); + + const loss = probe.train(); + return { + loss, + isNumber: typeof loss === 'number', + isNotNaN: loss !== null && !Number.isNaN(loss), + }; + }); + + expect(result.isNumber).toBe(true); + expect(result.isNotNaN).toBe(true); + }); + + // ─── VAL-ML-008: Training converges ─── + + test('VAL-ML-008a: second train() loss <= first train() loss', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Add several examples + probe.setInputs(0.1, 0.9); + store.addExample(); + + probe.setInputs(0.3, 0.7); + store.addExample(); + + probe.setInputs(0.5, 0.5); + store.addExample(); + + probe.setInputs(0.7, 0.3); + store.addExample(); + + probe.setInputs(0.9, 0.1); + store.addExample(); + + // First training + const loss1 = probe.train(); + + // Second training (same examples, should converge or stay same) + const loss2 = probe.train(); + + return { loss1, loss2 }; + }); + + expect(result.loss1).not.toBeNull(); + expect(result.loss2).not.toBeNull(); + // Second loss should be <= first (within epsilon) + expect(result.loss2!).toBeLessThanOrEqual(result.loss1! + 1e-6); + }); + + test('VAL-ML-008b: repeated training continues to converge', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Add examples + probe.setInputs(0.2, 0.8); + store.addExample(); + + probe.setInputs(0.8, 0.2); + store.addExample(); + + // Train multiple times and collect losses + const losses: (number | null)[] = []; + for (let i = 0; i < 5; i++) { + losses.push(probe.train()); + } + + return { losses }; + }); + + // All losses should be finite non-negative + for (const loss of result.losses) { + expect(loss).not.toBeNull(); + expect(Number.isFinite(loss as number)).toBe(true); + expect(loss as number).toBeGreaterThanOrEqual(0); + } + + // General trend: later losses <= earlier ones (allow small epsilon for numerical noise) + const first = result.losses[0] as number; + const last = result.losses[result.losses.length - 1] as number; + expect(last).toBeLessThanOrEqual(first + 1e-4); + }); + + // ─── VAL-ML-009: Async training via worker ─── + + test('VAL-ML-009a: trainAsync() returns a Promise', async ({ page }) => { + await loadSolidApp(page); + + const isPromise = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Add an example first + probe.setInputs(0.5, 0.5); + store.addExample(); + + const result = probe.trainAsync(); + return result instanceof Promise; + }); + + expect(isPromise).toBe(true); + }); + + test('VAL-ML-009b: trainAsync() resolves to finite loss', async ({ page }) => { + await loadSolidApp(page); + + const loss = await page.evaluate(async () => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Add examples + probe.setInputs(0.2, 0.8); + store.addExample(); + + probe.setInputs(0.5, 0.5); + store.addExample(); + + probe.setInputs(0.8, 0.2); + store.addExample(); + + const loss = await probe.trainAsync(); + return loss; + }); + + expect(loss).not.toBeNull(); + expect(typeof loss).toBe('number'); + expect(Number.isFinite(loss as number)).toBe(true); + expect(loss as number).toBeGreaterThanOrEqual(0); + }); + + test('VAL-ML-009c: trainAsync() returns null with no examples', async ({ page }) => { + await loadSolidApp(page); + + const loss = await page.evaluate(async () => { + const probe = (window as any).__nisps; + return await probe.trainAsync(); + }); + + expect(loss).toBeNull(); + }); + + test('VAL-ML-009d: trainAsync() updates outputs after completion', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Capture outputs before training at a specific position + probe.setInputs(0.3, 0.7); + const outputsAtInput = probe.getOutputs().slice(); + + // Add an example at a very different position with target outputs + // that differ from current behavior — this ensures training will change + // the outputs at our test position + store.addExample(); // captures at (0.3, 0.7) + probe.setInputs(0.8, 0.2); + store.addExample(); // captures at (0.8, 0.2) + + // Get outputs at (0.8, 0.2) before training + const beforeTraining = probe.getOutputs().slice(); + + const loss = await probe.trainAsync(); + + // After training, outputs at the same input position may have changed + const afterTraining = probe.getOutputs().slice(); + + return { + loss, + lossIsFinite: loss !== null && Number.isFinite(loss), + // Verify trainAsync completed and updated the IML state + outputsAfterTraining: afterTraining.length, + allBounded: afterTraining.every((v: number) => v >= 0 && v <= 1), + }; + }); + + expect(result.loss).not.toBeNull(); + expect(result.lossIsFinite).toBe(true); + // After async training, outputs should still be valid + expect(result.outputsAfterTraining).toBeGreaterThan(0); + expect(result.allBounded).toBe(true); + }); + + // ─── VAL-ML-010: Loss history populated after training ─── + + test('VAL-ML-010a: lossHistory has multiple entries after sync training', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + // Add examples + probe.setInputs(0.2, 0.8); + store.addExample(); + + probe.setInputs(0.8, 0.2); + store.addExample(); + + // Train twice to generate more loss history + probe.train(); + probe.train(); + + const history = probe.getLossHistory(); + return { + length: history.length, + allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0), + firstFew: history.slice(0, 5), + }; + }); + + // Loss history should have multiple entries (train_ex captures per-iteration losses) + expect(result.length).toBeGreaterThan(1); + expect(result.allFinite).toBe(true); + // All values should be non-negative + for (const v of result.firstFew) { + expect(v).toBeGreaterThanOrEqual(0); + } + }); + + test('VAL-ML-010b: lossHistory grows with each training call', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + probe.setInputs(0.3, 0.7); + store.addExample(); + + probe.setInputs(0.7, 0.3); + store.addExample(); + + const lengths: number[] = []; + + // Before training + lengths.push(probe.getLossHistory().length); + + // After first training + probe.train(); + lengths.push(probe.getLossHistory().length); + + // After second training + probe.train(); + lengths.push(probe.getLossHistory().length); + + return { lengths }; + }); + + // History should grow (or stay same) with each training call + expect(result.lengths[1]).toBeGreaterThanOrEqual(result.lengths[0]); + expect(result.lengths[2]).toBeGreaterThanOrEqual(result.lengths[1]); + // At least one entry should be added by training + expect(result.lengths[2]).toBeGreaterThan(0); + }); + + test('VAL-ML-010c: lossHistory populated after async training', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + const store = (window as any).__nispsStore; + + probe.setInputs(0.3, 0.7); + store.addExample(); + + probe.setInputs(0.7, 0.3); + store.addExample(); + + // Async train + await probe.trainAsync(); + + const history = probe.getLossHistory(); + return { + length: history.length, + allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0), + }; + }); + + expect(result.length).toBeGreaterThan(0); + expect(result.allFinite).toBe(true); + }); + + // ─── Cross-checks: thumbsUp triggers training ─── + + test('thumbsUp adds example AND triggers async training (loss updates)', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + + // Set initial position + probe.setInputs(0.3, 0.7); + + // ThumbsUp should add example + train async + const loss = await probe.thumbsUp(); + + return { + exampleCount: probe.getExampleCount(), + loss, + lossIsFinite: loss !== null && Number.isFinite(loss), + }; + }); + + expect(result.exampleCount).toBeGreaterThanOrEqual(1); + expect(result.lossIsFinite).toBe(true); + }); + + test('after thumbsUp, lossHistory has entries', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const probe = (window as any).__nisps; + + probe.setInputs(0.4, 0.6); + await probe.thumbsUp(); + + probe.setInputs(0.6, 0.4); + await probe.thumbsUp(); + + const history = probe.getLossHistory(); + return { + length: history.length, + allFinite: history.every((v: number) => Number.isFinite(v) && v >= 0), + }; + }); + + expect(result.length).toBeGreaterThan(0); + expect(result.allFinite).toBe(true); + }); +});