Bound inference result cache

src/inference/cache.ts

@@ -1,15 +1,27 @@
 import { hashText, normalizeText } from "../shared/hash";
 import type { InferenceResult } from "../shared/types";
 
+const DEFAULT_MAX_SIZE = 5_000;
+
 export class ResultCache {
   private readonly values = new Map<string, InferenceResult>();
 
+  constructor(private readonly maxSize = DEFAULT_MAX_SIZE) {
+    if (!Number.isInteger(maxSize) || maxSize < 1) {
+      throw new Error("Result cache max size must be a positive integer");
+    }
+  }
+
   get(text: string): InferenceResult | undefined {
     return this.values.get(hashText(text));
   }
 
   set(text: string, result: InferenceResult): void {
     this.values.set(hashText(normalizeText(text)), result);
+    if (this.values.size > this.maxSize) {
+      const oldest = this.values.keys().next().value;
+      if (oldest !== undefined) this.values.delete(oldest);
+    }
   }
 
   clear(): void {

tests/cache.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { ResultCache } from "../src/inference/cache";
+import type { InferenceResult } from "../src/shared/types";
+
+const result = (id: string): InferenceResult => ({
+  requestId: id,
+  id,
+  textHash: id,
+  label: "toxic",
+  probability: .9,
+  navigationId: "nav"
+});
+
+describe("ResultCache", () => {
+  it("evicts the oldest entry when the cache reaches its limit", () => {
+    const cache = new ResultCache(2);
+
+    cache.set("first", result("first"));
+    cache.set("second", result("second"));
+    cache.set("third", result("third"));
+
+    expect(cache.size).toBe(2);
+    expect(cache.get("first")).toBeUndefined();
+    expect(cache.get("second")?.id).toBe("second");
+    expect(cache.get("third")?.id).toBe("third");
+  });
+
+  it("uses normalized text for bounded cache entries", () => {
+    const cache = new ResultCache(1);
+
+    cache.set("  Same   text ", result("same"));
+
+    expect(cache.get("same text")?.id).toBe("same");
+    expect(cache.size).toBe(1);
+  });
+
+  it("rejects invalid cache sizes", () => {
+    expect(() => new ResultCache(0)).toThrow();
+    expect(() => new ResultCache(1.5)).toThrow();
+  });
+});