ml.predictbatch.test.ts

1// T-030: ml.ts predictBatch — batch sizes, softmax stability with extreme weights2//3// Why these tests exist: `predictBatch` in background/ml.ts is the ML inference4// entry point called by the autofill orchestration layer. It is also the5// critical path that runs on every form interaction. Two properties are untested:6//7//   1. Batch size handling: empty arrays, single items, and large batches8//      (100+ fields — enough to cover entire SPA page loads).9//10//   2. Softmax numerical stability: the softmax implementation uses the11//      max-subtraction trick (`x - max` before exp()) to prevent overflow for12//      extreme logit values. Without this, extremely large logits (e.g., 1000)13//      would produce `Infinity` in Math.exp(1000), making the confidence NaN.14//      This is the standard numerical stability fix and should be tested with15//      stress-case feature vectors.16//17// Note: the model weights (W, B) are baked in. We test the inference contract18// (output shape, probability constraints) not the model accuracy.1920import { describe, expect, it } from "vitest";21import { predictBatch } from "./ml";22import type { MLFieldFeatures, FieldPrediction, FieldType } from "../shared/ml";2324// ---------------------------------------------------------------------------25// Feature vector factory26// ---------------------------------------------------------------------------2728// Builds a zero-vector feature set (no signals → model returns "none" or29// uncertain distribution).30function zeroFeatures(): MLFieldFeatures {31  return {32    type_text: 0, type_email: 0, type_password: 0, type_tel: 0,33    type_number: 0, type_search: 0, type_url: 0, type_other: 0,34    auto_username: 0, auto_email: 0, auto_current_password: 0,35    auto_new_password: 0, auto_one_time_code: 0, auto_off: 0, auto_other: 0,36    name_has_user: 0, name_has_login: 0, name_has_email: 0, name_has_pass: 0,37    name_length: 0,38    id_has_user: 0, id_has_login: 0, id_has_email: 0, id_has_pass: 0,39    id_length: 0,40    placeholder_has_user: 0, placeholder_has_email: 0, placeholder_has_pass: 0,41    placeholder_length: 0,42    aria_label_has_user: 0, aria_label_has_email: 0, aria_label_has_pass: 0,43    aria_label_length: 0,44    parent_is_form: 0, parent_is_div: 0, parent_is_section: 0,45    sibling_count: 0, has_password_sibling: 0, has_email_sibling: 0,46    form_has_submit: 0, form_action_has_login: 0,47    is_required: 0, has_placeholder: 0, has_aria_label: 0,48    inputmode_numeric: 0,49  };50}5152// Builds features that strongly signal a password field.53function passwordFeatures(): MLFieldFeatures {54  return {55    ...zeroFeatures(),56    type_password: 1,57    auto_current_password: 1,58    name_has_pass: 1,59    id_has_pass: 1,60    parent_is_form: 1,61    form_has_submit: 1,62  };63}6465const VALID_TYPES: FieldType[] = ["username", "password", "email", "totp", "none"];6667function assertValidPrediction(p: FieldPrediction): void {68  expect(VALID_TYPES).toContain(p.type);69  expect(p.confidence).toBeGreaterThanOrEqual(0);70  expect(p.confidence).toBeLessThanOrEqual(1);71  expect(Number.isFinite(p.confidence)).toBe(true);72  expect(Number.isNaN(p.confidence)).toBe(false);7374  let sumProb = 0;75  for (const t of VALID_TYPES) {76    const prob = p.probabilities[t];77    expect(Number.isFinite(prob)).toBe(true);78    expect(prob).toBeGreaterThanOrEqual(0);79    sumProb += prob;80  }81  // Probabilities must sum to ~1.0 (floating point tolerance)82  expect(Math.abs(sumProb - 1.0)).toBeLessThan(1e-9);83}8485// ---------------------------------------------------------------------------86// Tests87// ---------------------------------------------------------------------------8889describe("T-030: predictBatch batch sizes", () => {90  // T-030a: empty array → returns empty array, no error.91  it("T-030a: predictBatch([]) returns empty array", () => {92    const result = predictBatch([]);93    expect(Array.isArray(result)).toBe(true);94    expect(result.length).toBe(0);95  });9697  // T-030b: single feature vector → returns exactly one prediction.98  it("T-030b: predictBatch([single]) returns exactly 1 prediction", () => {99    const result = predictBatch([zeroFeatures()]);100    expect(result.length).toBe(1);101    assertValidPrediction(result[0]!);102  });103104  // T-030c: batch of 100 feature vectors → returns exactly 100 predictions.105  // 100 fields covers a large SPA page load (e.g., a checkout form with106  // multiple steps rendered simultaneously).107  it("T-030c: predictBatch(100 vectors) returns 100 predictions", () => {108    const features = Array.from({ length: 100 }, () => zeroFeatures());109    const result = predictBatch(features);110    expect(result.length).toBe(100);111    for (const p of result) {112      assertValidPrediction(p);113    }114  });115116  // T-030d: password-signal features produce a prediction for "password" type.117  // This is not a model accuracy test — it just verifies that strong password118  // signals produce a non-zero confidence for the "password" class.119  it("T-030d: password-signal features → confidence for password is highest or meaningful", () => {120    const result = predictBatch([passwordFeatures()]);121    expect(result.length).toBe(1);122    assertValidPrediction(result[0]!);123    // With strong password features, the password probability should be notable124    expect(result[0]!.probabilities.password).toBeGreaterThan(0);125  });126});127128describe("T-030: predictBatch softmax numerical stability", () => {129  // T-030e: extreme positive logit values (all features = 1000) → softmax must130  // not produce NaN or Infinity. The max-subtraction trick prevents overflow.131  //132  // Bug this catches: removing the `x - max` subtraction in softmax would cause133  // Math.exp(1000) = Infinity, making all probabilities NaN and confidence NaN.134  it("T-030e: extreme positive feature values → no NaN or Infinity in output", () => {135    const extremeFeatures: MLFieldFeatures = {} as MLFieldFeatures;136    for (const key of Object.keys(zeroFeatures()) as (keyof MLFieldFeatures)[]) {137      (extremeFeatures as unknown as Record<string, number>)[key] = 1000;138    }139140    const result = predictBatch([extremeFeatures]);141    assertValidPrediction(result[0]!);142  });143144  // T-030f: extreme negative feature values → no NaN or Infinity.145  // Math.exp(-1000 - max) where max is also -1000 → exp(0) = 1 → stable.146  it("T-030f: extreme negative feature values → no NaN or Infinity", () => {147    const extremeNeg: MLFieldFeatures = {} as MLFieldFeatures;148    for (const key of Object.keys(zeroFeatures()) as (keyof MLFieldFeatures)[]) {149      (extremeNeg as unknown as Record<string, number>)[key] = -1000;150    }151152    const result = predictBatch([extremeNeg]);153    assertValidPrediction(result[0]!);154  });155156  // T-030g: mixed extreme values (some 1e6, some -1e6) → stable.157  it("T-030g: mixed extreme positive and negative feature values → stable softmax", () => {158    const mixed: MLFieldFeatures = {} as MLFieldFeatures;159    let i = 0;160    for (const key of Object.keys(zeroFeatures()) as (keyof MLFieldFeatures)[]) {161      (mixed as unknown as Record<string, number>)[key] = i % 2 === 0 ? 1e6 : -1e6;162      i++;163    }164165    const result = predictBatch([mixed]);166    assertValidPrediction(result[0]!);167  });168169  // T-030h: output type is always one of the 5 known FieldType values.170  // Bug this catches: CLASSES[maxIdx] would be undefined if maxIdx is out of171  // range (shouldn't happen, but the test pins the contract).172  it("T-030h: all predictions have a valid FieldType (not undefined)", () => {173    const cases = [174      zeroFeatures(),175      passwordFeatures(),176      { ...zeroFeatures(), type_email: 1, auto_email: 1, name_has_email: 1 },177      { ...zeroFeatures(), auto_one_time_code: 1, inputmode_numeric: 1 },178    ];179180    const results = predictBatch(cases);181    for (const r of results) {182      expect(VALID_TYPES).toContain(r.type);183    }184  });185});