Update AI response parsing for JSON reasoning

Generalizes parsing of response messages to account for multiple model response formats. Related to #1.

AGENTS.md

@@ -34,6 +34,7 @@ There are currently no automated tests for this project. If you add tests in the
 ## Endpoint Notes
 
 Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided.
+Responses are expected to include a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in the response text and ignores any surrounding commentary.
 
 ## Documentation
 

README.md

@@ -6,7 +6,10 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filt
 It allows you to classify email messages by sending their contents to a configurable
 HTTP endpoint. Sortana uses the `/v1/completions` API; the options page stores a base
 URL and appends `/v1/completions` when sending requests. The endpoint should respond
-with JSON indicating whether the message meets a specified criterion.
+with JSON indicating whether the message meets a specified criterion, including a
+short reasoning summary.
+Responses are parsed by extracting the last JSON object in the response text and
+expecting a `match` (or `matched`) boolean plus a `reason` string.
 
 ## Features
 

background.js

@@ -118,8 +118,17 @@ async function clearError() {
 }
 
 function recordError(context, err) {
-    const message = err instanceof Error ? err.message : String(err || 'Unknown error');
-    const detail = err instanceof Error ? err.stack : '';
+    let message = 'Unknown error';
+    let detail = '';
+    if (err instanceof Error) {
+        message = err.message;
+        detail = err.stack || '';
+    } else if (err && typeof err === 'object') {
+        message = typeof err.message === 'string' ? err.message : String(err || 'Unknown error');
+        detail = typeof err.detail === 'string' ? err.detail : '';
+    } else {
+        message = String(err || 'Unknown error');
+    }
     errorLog.unshift({
         time: Date.now(),
         context,
@@ -741,6 +750,9 @@ async function clearCacheForMessages(idsInput) {
             return { count: queuedCount + (processing ? 1 : 0) };
         } else if (msg?.type === "sortana:getErrorLog") {
             return { errors: errorLog.slice() };
+        } else if (msg?.type === "sortana:recordError") {
+            recordError(msg.context || "Sortana Error", { message: msg.message, detail: msg.detail });
+            return { ok: true };
         } else if (msg?.type === "sortana:getTiming") {
             const t = timingStats;
             const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0;

modules/AiClassifier.js

@@ -25,8 +25,8 @@ const DEFAULT_CUSTOM_SYSTEM_PROMPT = "Determine whether the email satisfies the
 
 const SYSTEM_SUFFIX = `
 Return ONLY a JSON object on a single line of the form:
-{"match": true} - if the email satisfies the criterion
-{"match": false} - otherwise
+{"match": true, "reason": "<short explanation>"} - if the email satisfies the criterion
+{"match": false, "reason": "<short explanation>"} - otherwise
 
 Do not add any other keys, text, or formatting.`;
 
@@ -266,15 +266,95 @@ function buildPayload(text, criterion) {
   return JSON.stringify(payloadObj);
 }
 
+function reportParseError(message, detail) {
+  try {
+    const runtime = (globalThis.browser ?? globalThis.messenger)?.runtime;
+    if (!runtime?.sendMessage) {
+      return;
+    }
+    runtime.sendMessage({
+      type: "sortana:recordError",
+      context: "AI response parsing",
+      message,
+      detail
+    }).catch(() => {});
+  } catch (e) {
+    aiLog("Failed to report parse error", { level: "warn" }, e);
+  }
+}
+
+function extractLastJsonObject(text) {
+  let last = null;
+  let start = -1;
+  let depth = 0;
+  let inString = false;
+  let escape = false;
+
+  for (let i = 0; i < text.length; i += 1) {
+    const ch = text[i];
+    if (inString) {
+      if (escape) {
+        escape = false;
+        continue;
+      }
+      if (ch === "\\") {
+        escape = true;
+        continue;
+      }
+      if (ch === "\"") {
+        inString = false;
+      }
+      continue;
+    }
+    if (ch === "\"") {
+      inString = true;
+      continue;
+    }
+    if (ch === "{") {
+      if (depth === 0) {
+        start = i;
+      }
+      depth += 1;
+      continue;
+    }
+    if (ch === "}" && depth > 0) {
+      depth -= 1;
+      if (depth === 0 && start !== -1) {
+        last = text.slice(start, i + 1);
+        start = -1;
+      }
+    }
+  }
+
+  return last;
+}
+
 function parseMatch(result) {
   const rawText = result.choices?.[0]?.text || "";
-  const thinkText = rawText.match(/<think>[\s\S]*?<\/think>/gi)?.join('') || '';
-  aiLog('[AiClassifier] ⮡ Reasoning:', {debug: true}, thinkText);
-  const cleanedText = rawText.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
-  aiLog('[AiClassifier] ⮡ Cleaned Response Text:', {debug: true}, cleanedText);
-  const obj = JSON.parse(cleanedText);
-  const matched = obj.matched === true || obj.match === true;
-  return { matched, reason: thinkText };
+  const candidate = extractLastJsonObject(rawText);
+  if (!candidate) {
+    reportParseError("No JSON object found in AI response.", rawText.slice(0, 800));
+    return { matched: false, reason: "" };
+  }
+
+  let obj;
+  try {
+    obj = JSON.parse(candidate);
+  } catch (e) {
+    reportParseError("Failed to parse JSON from AI response.", candidate.slice(0, 800));
+    return { matched: false, reason: "" };
+  }
+
+  const matchValue = Object.prototype.hasOwnProperty.call(obj, "match") ? obj.match : obj.matched;
+  const matched = matchValue === true;
+  if (matchValue !== true && matchValue !== false) {
+    reportParseError("AI response missing valid match boolean.", candidate.slice(0, 800));
+  }
+
+  const reasonValue = obj.reason ?? obj.reasoning ?? obj.explaination;
+  const reason = typeof reasonValue === "string" ? reasonValue : "";
+
+  return { matched, reason };
 }
 
 function cacheEntry(cacheKey, matched, reason) {

prompt_templates/mistral.txt

@@ -5,8 +5,8 @@ Email:
 
 Criterion: {{query}}
 Remember, return ONLY a JSON object on a single line of the form:
-{"match": true} - if the email satisfies the criterion
-{"match": false} - otherwise
+{"match": true, "reason": "<short explanation>"} - if the email satisfies the criterion
+{"match": false, "reason": "<short explanation>"} - otherwise
 
 Do not add any other keys, text, or formatting.
 [/INST]

prompt_templates/openai.txt

@@ -7,8 +7,8 @@
 ```
 Classification Criterion: {{query}}
 Remember, return ONLY a JSON object on a single line of the form:
-{"match": true} - if the email satisfies the criterion
-{"match": false} - otherwise
+{"match": true, "reason": "<short explanation>"} - if the email satisfies the criterion
+{"match": false, "reason": "<short explanation>"} - otherwise
 
 Do not add any other keys, text, or formatting.<|im_end|>
 <|im_start|>assistant

prompt_templates/qwen.txt

@@ -7,8 +7,8 @@ Email:
 
 Criterion: {{query}}
 Remember, return ONLY a JSON object on a single line of the form:
-{"match": true} - if the email satisfies the criterion
-{"match": false} - otherwise
+{"match": true, "reason": "<short explanation>"} - if the email satisfies the criterion
+{"match": false, "reason": "<short explanation>"} - otherwise
 
 Do not add any other keys, text, or formatting.
 <|im_end|>