From 42d013fccd0090816111f5032c1aea85b426ac30 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Wed, 7 Jan 2026 03:32:29 -0600 Subject: [PATCH] Update AI response parsing for JSON reasoning --- AGENTS.md | 1 + README.md | 5 +- background.js | 16 +++++- modules/AiClassifier.js | 98 ++++++++++++++++++++++++++++++++---- prompt_templates/mistral.txt | 4 +- prompt_templates/openai.txt | 4 +- prompt_templates/qwen.txt | 4 +- 7 files changed, 114 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 23080ae..5f5f2f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,6 +34,7 @@ There are currently no automated tests for this project. If you add tests in the ## Endpoint Notes Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. +Responses are expected to include a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in the response text and ignores any surrounding commentary. ## Documentation diff --git a/README.md b/README.md index 87a3565..aec6884 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,10 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filt It allows you to classify email messages by sending their contents to a configurable HTTP endpoint. Sortana uses the `/v1/completions` API; the options page stores a base URL and appends `/v1/completions` when sending requests. The endpoint should respond -with JSON indicating whether the message meets a specified criterion. +with JSON indicating whether the message meets a specified criterion, including a +short reasoning summary. +Responses are parsed by extracting the last JSON object in the response text and +expecting a `match` (or `matched`) boolean plus a `reason` string. ## Features diff --git a/background.js b/background.js index e16d5a3..64aad50 100644 --- a/background.js +++ b/background.js @@ -118,8 +118,17 @@ async function clearError() { } function recordError(context, err) { - const message = err instanceof Error ? err.message : String(err || 'Unknown error'); - const detail = err instanceof Error ? err.stack : ''; + let message = 'Unknown error'; + let detail = ''; + if (err instanceof Error) { + message = err.message; + detail = err.stack || ''; + } else if (err && typeof err === 'object') { + message = typeof err.message === 'string' ? err.message : String(err || 'Unknown error'); + detail = typeof err.detail === 'string' ? err.detail : ''; + } else { + message = String(err || 'Unknown error'); + } errorLog.unshift({ time: Date.now(), context, @@ -741,6 +750,9 @@ async function clearCacheForMessages(idsInput) { return { count: queuedCount + (processing ? 1 : 0) }; } else if (msg?.type === "sortana:getErrorLog") { return { errors: errorLog.slice() }; + } else if (msg?.type === "sortana:recordError") { + recordError(msg.context || "Sortana Error", { message: msg.message, detail: msg.detail }); + return { ok: true }; } else if (msg?.type === "sortana:getTiming") { const t = timingStats; const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0; diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index f5a3bff..603c088 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -25,8 +25,8 @@ const DEFAULT_CUSTOM_SYSTEM_PROMPT = "Determine whether the email satisfies the const SYSTEM_SUFFIX = ` Return ONLY a JSON object on a single line of the form: -{"match": true} - if the email satisfies the criterion -{"match": false} - otherwise +{"match": true, "reason": ""} - if the email satisfies the criterion +{"match": false, "reason": ""} - otherwise Do not add any other keys, text, or formatting.`; @@ -266,15 +266,95 @@ function buildPayload(text, criterion) { return JSON.stringify(payloadObj); } +function reportParseError(message, detail) { + try { + const runtime = (globalThis.browser ?? globalThis.messenger)?.runtime; + if (!runtime?.sendMessage) { + return; + } + runtime.sendMessage({ + type: "sortana:recordError", + context: "AI response parsing", + message, + detail + }).catch(() => {}); + } catch (e) { + aiLog("Failed to report parse error", { level: "warn" }, e); + } +} + +function extractLastJsonObject(text) { + let last = null; + let start = -1; + let depth = 0; + let inString = false; + let escape = false; + + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (inString) { + if (escape) { + escape = false; + continue; + } + if (ch === "\\") { + escape = true; + continue; + } + if (ch === "\"") { + inString = false; + } + continue; + } + if (ch === "\"") { + inString = true; + continue; + } + if (ch === "{") { + if (depth === 0) { + start = i; + } + depth += 1; + continue; + } + if (ch === "}" && depth > 0) { + depth -= 1; + if (depth === 0 && start !== -1) { + last = text.slice(start, i + 1); + start = -1; + } + } + } + + return last; +} + function parseMatch(result) { const rawText = result.choices?.[0]?.text || ""; - const thinkText = rawText.match(/[\s\S]*?<\/think>/gi)?.join('') || ''; - aiLog('[AiClassifier] ⮡ Reasoning:', {debug: true}, thinkText); - const cleanedText = rawText.replace(/[\s\S]*?<\/think>/gi, "").trim(); - aiLog('[AiClassifier] ⮡ Cleaned Response Text:', {debug: true}, cleanedText); - const obj = JSON.parse(cleanedText); - const matched = obj.matched === true || obj.match === true; - return { matched, reason: thinkText }; + const candidate = extractLastJsonObject(rawText); + if (!candidate) { + reportParseError("No JSON object found in AI response.", rawText.slice(0, 800)); + return { matched: false, reason: "" }; + } + + let obj; + try { + obj = JSON.parse(candidate); + } catch (e) { + reportParseError("Failed to parse JSON from AI response.", candidate.slice(0, 800)); + return { matched: false, reason: "" }; + } + + const matchValue = Object.prototype.hasOwnProperty.call(obj, "match") ? obj.match : obj.matched; + const matched = matchValue === true; + if (matchValue !== true && matchValue !== false) { + reportParseError("AI response missing valid match boolean.", candidate.slice(0, 800)); + } + + const reasonValue = obj.reason ?? obj.reasoning ?? obj.explaination; + const reason = typeof reasonValue === "string" ? reasonValue : ""; + + return { matched, reason }; } function cacheEntry(cacheKey, matched, reason) { diff --git a/prompt_templates/mistral.txt b/prompt_templates/mistral.txt index f78bed7..1ee56fa 100644 --- a/prompt_templates/mistral.txt +++ b/prompt_templates/mistral.txt @@ -5,8 +5,8 @@ Email: Criterion: {{query}} Remember, return ONLY a JSON object on a single line of the form: -{"match": true} - if the email satisfies the criterion -{"match": false} - otherwise +{"match": true, "reason": ""} - if the email satisfies the criterion +{"match": false, "reason": ""} - otherwise Do not add any other keys, text, or formatting. [/INST] diff --git a/prompt_templates/openai.txt b/prompt_templates/openai.txt index bf7cea1..c770394 100644 --- a/prompt_templates/openai.txt +++ b/prompt_templates/openai.txt @@ -7,8 +7,8 @@ ``` Classification Criterion: {{query}} Remember, return ONLY a JSON object on a single line of the form: -{"match": true} - if the email satisfies the criterion -{"match": false} - otherwise +{"match": true, "reason": ""} - if the email satisfies the criterion +{"match": false, "reason": ""} - otherwise Do not add any other keys, text, or formatting.<|im_end|> <|im_start|>assistant diff --git a/prompt_templates/qwen.txt b/prompt_templates/qwen.txt index 0c25472..1aa0ef0 100644 --- a/prompt_templates/qwen.txt +++ b/prompt_templates/qwen.txt @@ -7,8 +7,8 @@ Email: Criterion: {{query}} Remember, return ONLY a JSON object on a single line of the form: -{"match": true} - if the email satisfies the criterion -{"match": false} - otherwise +{"match": true, "reason": ""} - if the email satisfies the criterion +{"match": false, "reason": ""} - otherwise Do not add any other keys, text, or formatting. <|im_end|>