// T-030: ml.ts predictBatch — batch sizes, softmax stability with extreme weights // // Why these tests exist: `predictBatch` in background/ml.ts is the ML inference // entry point called by the autofill orchestration layer. It is also the // critical path that runs on every form interaction. Two properties are untested: // // 1. Batch size handling: empty arrays, single items, and large batches // (100+ fields — enough to cover entire SPA page loads). // // 2. Softmax numerical stability: the softmax implementation uses the // max-subtraction trick (`x - max` before exp()) to prevent overflow for // extreme logit values. Without this, extremely large logits (e.g., 1000) // would produce `Infinity` in Math.exp(1000), making the confidence NaN. // This is the standard numerical stability fix and should be tested with // stress-case feature vectors. // // Note: the model weights (W, B) are baked in. We test the inference contract // (output shape, probability constraints) not the model accuracy. import { describe, expect, it } from "vitest"; import { predictBatch } from "./ml"; import type { MLFieldFeatures, FieldPrediction, FieldType } from "../shared/ml"; // --------------------------------------------------------------------------- // Feature vector factory // --------------------------------------------------------------------------- // Builds a zero-vector feature set (no signals → model returns "none" or // uncertain distribution). function zeroFeatures(): MLFieldFeatures { return { type_text: 0, type_email: 0, type_password: 0, type_tel: 0, type_number: 0, type_search: 0, type_url: 0, type_other: 0, auto_username: 0, auto_email: 0, auto_current_password: 0, auto_new_password: 0, auto_one_time_code: 0, auto_off: 0, auto_other: 0, name_has_user: 0, name_has_login: 0, name_has_email: 0, name_has_pass: 0, name_length: 0, id_has_user: 0, id_has_login: 0, id_has_email: 0, id_has_pass: 0, id_length: 0, placeholder_has_user: 0, placeholder_has_email: 0, placeholder_has_pass: 0, placeholder_length: 0, aria_label_has_user: 0, aria_label_has_email: 0, aria_label_has_pass: 0, aria_label_length: 0, parent_is_form: 0, parent_is_div: 0, parent_is_section: 0, sibling_count: 0, has_password_sibling: 0, has_email_sibling: 0, form_has_submit: 0, form_action_has_login: 0, is_required: 0, has_placeholder: 0, has_aria_label: 0, inputmode_numeric: 0, }; } // Builds features that strongly signal a password field. function passwordFeatures(): MLFieldFeatures { return { ...zeroFeatures(), type_password: 1, auto_current_password: 1, name_has_pass: 1, id_has_pass: 1, parent_is_form: 1, form_has_submit: 1, }; } const VALID_TYPES: FieldType[] = ["username", "password", "email", "totp", "none"]; function assertValidPrediction(p: FieldPrediction): void { expect(VALID_TYPES).toContain(p.type); expect(p.confidence).toBeGreaterThanOrEqual(0); expect(p.confidence).toBeLessThanOrEqual(1); expect(Number.isFinite(p.confidence)).toBe(true); expect(Number.isNaN(p.confidence)).toBe(false); let sumProb = 0; for (const t of VALID_TYPES) { const prob = p.probabilities[t]; expect(Number.isFinite(prob)).toBe(true); expect(prob).toBeGreaterThanOrEqual(0); sumProb += prob; } // Probabilities must sum to ~1.0 (floating point tolerance) expect(Math.abs(sumProb - 1.0)).toBeLessThan(1e-9); } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- describe("T-030: predictBatch batch sizes", () => { // T-030a: empty array → returns empty array, no error. it("T-030a: predictBatch([]) returns empty array", () => { const result = predictBatch([]); expect(Array.isArray(result)).toBe(true); expect(result.length).toBe(0); }); // T-030b: single feature vector → returns exactly one prediction. it("T-030b: predictBatch([single]) returns exactly 1 prediction", () => { const result = predictBatch([zeroFeatures()]); expect(result.length).toBe(1); assertValidPrediction(result[0]!); }); // T-030c: batch of 100 feature vectors → returns exactly 100 predictions. // 100 fields covers a large SPA page load (e.g., a checkout form with // multiple steps rendered simultaneously). it("T-030c: predictBatch(100 vectors) returns 100 predictions", () => { const features = Array.from({ length: 100 }, () => zeroFeatures()); const result = predictBatch(features); expect(result.length).toBe(100); for (const p of result) { assertValidPrediction(p); } }); // T-030d: password-signal features produce a prediction for "password" type. // This is not a model accuracy test — it just verifies that strong password // signals produce a non-zero confidence for the "password" class. it("T-030d: password-signal features → confidence for password is highest or meaningful", () => { const result = predictBatch([passwordFeatures()]); expect(result.length).toBe(1); assertValidPrediction(result[0]!); // With strong password features, the password probability should be notable expect(result[0]!.probabilities.password).toBeGreaterThan(0); }); }); describe("T-030: predictBatch softmax numerical stability", () => { // T-030e: extreme positive logit values (all features = 1000) → softmax must // not produce NaN or Infinity. The max-subtraction trick prevents overflow. // // Bug this catches: removing the `x - max` subtraction in softmax would cause // Math.exp(1000) = Infinity, making all probabilities NaN and confidence NaN. it("T-030e: extreme positive feature values → no NaN or Infinity in output", () => { const extremeFeatures: MLFieldFeatures = {} as MLFieldFeatures; for (const key of Object.keys(zeroFeatures()) as (keyof MLFieldFeatures)[]) { (extremeFeatures as unknown as Record)[key] = 1000; } const result = predictBatch([extremeFeatures]); assertValidPrediction(result[0]!); }); // T-030f: extreme negative feature values → no NaN or Infinity. // Math.exp(-1000 - max) where max is also -1000 → exp(0) = 1 → stable. it("T-030f: extreme negative feature values → no NaN or Infinity", () => { const extremeNeg: MLFieldFeatures = {} as MLFieldFeatures; for (const key of Object.keys(zeroFeatures()) as (keyof MLFieldFeatures)[]) { (extremeNeg as unknown as Record)[key] = -1000; } const result = predictBatch([extremeNeg]); assertValidPrediction(result[0]!); }); // T-030g: mixed extreme values (some 1e6, some -1e6) → stable. it("T-030g: mixed extreme positive and negative feature values → stable softmax", () => { const mixed: MLFieldFeatures = {} as MLFieldFeatures; let i = 0; for (const key of Object.keys(zeroFeatures()) as (keyof MLFieldFeatures)[]) { (mixed as unknown as Record)[key] = i % 2 === 0 ? 1e6 : -1e6; i++; } const result = predictBatch([mixed]); assertValidPrediction(result[0]!); }); // T-030h: output type is always one of the 5 known FieldType values. // Bug this catches: CLASSES[maxIdx] would be undefined if maxIdx is out of // range (shouldn't happen, but the test pins the contract). it("T-030h: all predictions have a valid FieldType (not undefined)", () => { const cases = [ zeroFeatures(), passwordFeatures(), { ...zeroFeatures(), type_email: 1, auto_email: 1, name_has_email: 1 }, { ...zeroFeatures(), auto_one_time_code: 1, inputmode_numeric: 1 }, ]; const results = predictBatch(cases); for (const r of results) { expect(VALID_TYPES).toContain(r.type); } }); });