From fa84f37637e1bc5bd012ca565987c0f0ab180a74 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Tue, 15 Jul 2025 23:07:55 -0500 Subject: [PATCH 01/44] Add rule enable toggle --- README.md | 5 +++-- background.js | 5 +++++ options/options.js | 20 +++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index adebc99..60cbe13 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ message meets a specified criterion. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. +- **Rule enable/disable** – temporarily turn a rule off without removing it. - **Account & folder filters** – limit rules to specific accounts or folders. - **Context menu** – apply AI rules from the message list or the message display action button. - **Status icons** – toolbar icons show when classification is in progress and briefly display success or error states. @@ -75,8 +76,8 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e deleting or archiving a message when it matches. Drag rules to reorder them, check *Only apply to unread messages* to skip read mail, set optional minimum or maximum message age limits, select the accounts or - folders a rule should apply to, and - check *Stop after match* to halt further processing. Forward and reply actions + folders a rule should apply to, uncheck *Enabled* to temporarily disable a rule, and + check *Stop after match* to halt further processing. Forward and reply actions open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the configured rules. diff --git a/background.js b/background.js index fb820bb..b0db614 100644 --- a/background.js +++ b/background.js @@ -36,6 +36,7 @@ function normalizeRules(rules) { if (r.actions) { if (!Array.isArray(r.accounts)) r.accounts = []; if (!Array.isArray(r.folders)) r.folders = []; + r.enabled = r.enabled !== false; return r; } const actions = []; @@ -49,6 +50,7 @@ function normalizeRules(rules) { if (typeof r.maxAgeDays === 'number') rule.maxAgeDays = r.maxAgeDays; if (Array.isArray(r.accounts)) rule.accounts = r.accounts; if (Array.isArray(r.folders)) rule.folders = r.folders; + rule.enabled = r.enabled !== false; return rule; }) : []; } @@ -234,6 +236,9 @@ async function processMessage(id) { } for (const rule of aiRules) { + if (rule.enabled === false) { + continue; + } if (hdr && Array.isArray(rule.accounts) && rule.accounts.length && !rule.accounts.includes(hdr.folder.accountId)) { continue; diff --git a/options/options.js b/options/options.js index 0042004..3d02639 100644 --- a/options/options.js +++ b/options/options.js @@ -326,6 +326,16 @@ document.addEventListener('DOMContentLoaded', async () => { }); header.appendChild(delBtn); + const enabledLabel = document.createElement('label'); + enabledLabel.className = 'checkbox ml-2'; + const enabledCheck = document.createElement('input'); + enabledCheck.type = 'checkbox'; + enabledCheck.className = 'rule-enabled'; + enabledCheck.checked = rule.enabled !== false; + enabledLabel.appendChild(enabledCheck); + enabledLabel.append(' Enabled'); + header.appendChild(enabledLabel); + const actionsContainer = document.createElement('div'); actionsContainer.className = 'rule-actions mb-2'; @@ -469,18 +479,19 @@ document.addEventListener('DOMContentLoaded', async () => { }); const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; const unreadOnly = ruleEl.querySelector('.unread-only')?.checked; + const enabled = ruleEl.querySelector('.rule-enabled')?.checked !== false; const minAgeDays = parseFloat(ruleEl.querySelector('.min-age')?.value); const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, unreadOnly, stopProcessing }; + const rule = { criterion, actions, unreadOnly, stopProcessing, enabled }; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; if (accounts.length) rule.accounts = accounts; if (folders.length) rule.folders = folders; return rule; }); - data.push({ criterion: '', actions: [], unreadOnly: false, stopProcessing: false, accounts: [], folders: [] }); + data.push({ criterion: '', actions: [], unreadOnly: false, stopProcessing: false, enabled: true, accounts: [], folders: [] }); renderRules(data); }); @@ -488,6 +499,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (r.actions) { if (!Array.isArray(r.accounts)) r.accounts = []; if (!Array.isArray(r.folders)) r.folders = []; + if (r.enabled !== false) r.enabled = true; else r.enabled = false; return r; } const actions = []; @@ -501,6 +513,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (typeof r.maxAgeDays === 'number') rule.maxAgeDays = r.maxAgeDays; if (Array.isArray(r.accounts)) rule.accounts = r.accounts; if (Array.isArray(r.folders)) rule.folders = r.folders; + rule.enabled = r.enabled !== false; return rule; })); @@ -644,11 +657,12 @@ document.addEventListener('DOMContentLoaded', async () => { }); const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; const unreadOnly = ruleEl.querySelector('.unread-only')?.checked; + const enabled = ruleEl.querySelector('.rule-enabled')?.checked !== false; const minAgeDays = parseFloat(ruleEl.querySelector('.min-age')?.value); const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, unreadOnly, stopProcessing }; + const rule = { criterion, actions, unreadOnly, stopProcessing, enabled }; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; if (accounts.length) rule.accounts = accounts; From 65ff743676f152e93d6f939bf96ffde158b0fe44 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Tue, 15 Jul 2025 23:16:43 -0500 Subject: [PATCH 02/44] Persist error icon and add notification --- README.md | 6 ++++-- background.js | 50 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 60cbe13..9c75c61 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ message meets a specified criterion. - **Rule enable/disable** – temporarily turn a rule off without removing it. - **Account & folder filters** – limit rules to specific accounts or folders. - **Context menu** – apply AI rules from the message list or the message display action button. -- **Status icons** – toolbar icons show when classification is in progress and briefly display success or error states. +- **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red until you dismiss the notification. +- **Error notification** – failed classification displays a notification with a button to clear the error and reset the icon. - **View reasoning** – inspect why rules matched via the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. @@ -76,11 +77,12 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e deleting or archiving a message when it matches. Drag rules to reorder them, check *Only apply to unread messages* to skip read mail, set optional minimum or maximum message age limits, select the accounts or - folders a rule should apply to, uncheck *Enabled* to temporarily disable a rule, and + folders a rule should apply to, uncheck *Enabled* to temporarily disable a rule, and check *Stop after match* to halt further processing. Forward and reply actions open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the configured rules. +4. If the toolbar icon shows a red X, click the notification's **Dismiss** button to clear the error. ### Example Filters diff --git a/background.js b/background.js index b0db614..06683e2 100644 --- a/background.js +++ b/background.js @@ -30,6 +30,8 @@ let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; let detectSystemTheme; +let errorPending = false; +const ERROR_NOTIFICATION_ID = 'sortana-error'; function normalizeRules(rules) { return Array.isArray(rules) ? rules.map(r => { @@ -68,7 +70,8 @@ const ICONS = { logo: () => 'resources/img/logo.png', circledots: () => iconPaths('circledots'), circle: () => iconPaths('circle'), - average: () => iconPaths('average') + average: () => iconPaths('average'), + error: () => iconPaths('x') }; function setIcon(path) { @@ -82,19 +85,31 @@ function setIcon(path) { function updateActionIcon() { let path = ICONS.logo(); - if (processing || queuedCount > 0) { + if (errorPending) { + path = ICONS.error(); + } else if (processing || queuedCount > 0) { path = ICONS.circledots(); } setIcon(path); } function showTransientIcon(factory, delay = 1500) { + if (errorPending) { + return; + } clearTimeout(iconTimer); const path = typeof factory === 'function' ? factory() : factory; setIcon(path); iconTimer = setTimeout(updateActionIcon, delay); } +async function clearError() { + errorPending = false; + await storage.local.set({ errorPending: false }); + await browser.notifications.clear(ERROR_NOTIFICATION_ID); + updateActionIcon(); +} + function refreshMenuIcons() { browser.menus.update('apply-ai-rules-list', { icons: iconPaths('eye') }); browser.menus.update('apply-ai-rules-display', { icons: iconPaths('eye') }); @@ -307,9 +322,17 @@ async function processMessage(id) { const elapsed = Date.now() - currentStart; currentStart = 0; updateTimingStats(elapsed); - await storage.local.set({ classifyStats: timingStats }); + await storage.local.set({ classifyStats: timingStats, errorPending: true }); + errorPending = true; logger.aiLog("failed to apply AI rules", { level: 'error' }, e); - showTransientIcon(ICONS.average); + setIcon(ICONS.error()); + browser.notifications.create(ERROR_NOTIFICATION_ID, { + type: 'basic', + iconUrl: browser.runtime.getURL('resources/img/logo.png'), + title: 'Sortana Error', + message: 'Failed to apply AI rules', + buttons: [{ title: 'Dismiss' }] + }); } } async function applyAiRules(idsInput) { @@ -368,7 +391,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "aiRules", "theme"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "aiRules", "theme", "errorPending"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -378,6 +401,7 @@ async function clearCacheForMessages(idsInput) { stripUrlParams = store.stripUrlParams === true; altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; + errorPending = store.errorPending === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { Object.assign(timingStats, savedStats.classifyStats); @@ -409,6 +433,10 @@ async function clearCacheForMessages(idsInput) { collapseWhitespace = changes.collapseWhitespace.newValue === true; logger.aiLog("collapseWhitespace updated from storage change", { debug: true }, collapseWhitespace); } + if (changes.errorPending) { + errorPending = changes.errorPending.newValue === true; + updateActionIcon(); + } if (changes.theme) { userTheme = changes.theme.newValue || 'auto'; currentTheme = userTheme === 'auto' ? await detectSystemTheme() : userTheme; @@ -634,6 +662,18 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("Unhandled promise rejection", { level: 'error' }, ev.reason); }); + browser.notifications.onClicked.addListener(id => { + if (id === ERROR_NOTIFICATION_ID) { + clearError(); + } + }); + + browser.notifications.onButtonClicked.addListener((id) => { + if (id === ERROR_NOTIFICATION_ID) { + clearError(); + } + }); + browser.runtime.onInstalled.addListener(async ({ reason }) => { if (reason === "install") { await browser.runtime.openOptionsPage(); From 8bfd00dc493fe371ae0ac09527bef7b38ee4f0c8 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Tue, 15 Jul 2025 23:44:43 -0500 Subject: [PATCH 03/44] Added svg --- resources/svg/circleslash.svg | 3 +++ resources/svg/plus.svg | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 resources/svg/circleslash.svg create mode 100644 resources/svg/plus.svg diff --git a/resources/svg/circleslash.svg b/resources/svg/circleslash.svg new file mode 100644 index 0000000..861352d --- /dev/null +++ b/resources/svg/circleslash.svg @@ -0,0 +1,3 @@ + + + diff --git a/resources/svg/plus.svg b/resources/svg/plus.svg new file mode 100644 index 0000000..e665b91 --- /dev/null +++ b/resources/svg/plus.svg @@ -0,0 +1,3 @@ + + + From 0cddb5b204a1ac23d614835b5d2481c40dff4206 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Tue, 15 Jul 2025 23:50:49 -0500 Subject: [PATCH 04/44] Use SVG icons for rule controls and remove extra PNGs --- README.md | 3 +- options/options.js | 71 +++++++++++++++++++++++++++-------- resources/svg/circleslash.svg | 4 ++ 3 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 resources/svg/circleslash.svg diff --git a/README.md b/README.md index 9c75c61..7637f85 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,8 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e deleting or archiving a message when it matches. Drag rules to reorder them, check *Only apply to unread messages* to skip read mail, set optional minimum or maximum message age limits, select the accounts or - folders a rule should apply to, uncheck *Enabled* to temporarily disable a rule, and + folders a rule should apply to. Use the + slashed-circle/check button to disable or re-enable a rule, and check *Stop after match* to halt further processing. Forward and reply actions open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the diff --git a/options/options.js b/options/options.js index 3d02639..68b58da 100644 --- a/options/options.js +++ b/options/options.js @@ -315,26 +315,67 @@ document.addEventListener('DOMContentLoaded', async () => { const header = document.createElement('div'); header.className = 'message-header'; - header.appendChild(critInput); + + const leftWrap = document.createElement('div'); + leftWrap.style.display = 'flex'; + leftWrap.style.alignItems = 'center'; + leftWrap.style.flexGrow = '1'; + + const statusSpan = document.createElement('span'); + statusSpan.className = 'rule-status has-text-weight-semibold mr-2'; + + leftWrap.appendChild(statusSpan); + leftWrap.appendChild(critInput); + header.appendChild(leftWrap); + + const btnWrap = document.createElement('div'); + btnWrap.style.display = 'flex'; + btnWrap.style.gap = '0.25em'; + + let enabled = rule.enabled !== false; + + const toggleBtn = document.createElement('button'); + toggleBtn.type = 'button'; + toggleBtn.className = 'button is-small rule-toggle'; + const toggleIcon = document.createElement('img'); + toggleIcon.width = 16; + toggleIcon.height = 16; + toggleBtn.appendChild(toggleIcon); const delBtn = document.createElement('button'); - delBtn.className = 'delete'; - delBtn.setAttribute('aria-label', 'delete'); + delBtn.type = 'button'; + delBtn.className = 'button is-small is-danger is-light rule-delete'; + const delIcon = document.createElement('img'); + delIcon.src = browser.runtime.getURL('resources/svg/trash.svg'); + delIcon.width = 16; + delIcon.height = 16; + delBtn.appendChild(delIcon); + + function updateToggle() { + toggleIcon.src = browser.runtime.getURL( + `resources/svg/${enabled ? 'circleslash' : 'check'}.svg` + ); + statusSpan.textContent = enabled ? '' : '(Disabled)'; + article.dataset.enabled = String(enabled); + } + + toggleBtn.addEventListener('click', () => { + enabled = !enabled; + markDirty(); + updateToggle(); + }); + delBtn.addEventListener('click', () => { article.remove(); ruleCountEl.textContent = rulesContainer.querySelectorAll('.rule').length; + markDirty(); }); - header.appendChild(delBtn); - const enabledLabel = document.createElement('label'); - enabledLabel.className = 'checkbox ml-2'; - const enabledCheck = document.createElement('input'); - enabledCheck.type = 'checkbox'; - enabledCheck.className = 'rule-enabled'; - enabledCheck.checked = rule.enabled !== false; - enabledLabel.appendChild(enabledCheck); - enabledLabel.append(' Enabled'); - header.appendChild(enabledLabel); + btnWrap.appendChild(toggleBtn); + btnWrap.appendChild(delBtn); + header.appendChild(btnWrap); + + updateToggle(); const actionsContainer = document.createElement('div'); actionsContainer.className = 'rule-actions mb-2'; @@ -479,7 +520,7 @@ document.addEventListener('DOMContentLoaded', async () => { }); const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; const unreadOnly = ruleEl.querySelector('.unread-only')?.checked; - const enabled = ruleEl.querySelector('.rule-enabled')?.checked !== false; + const enabled = ruleEl.dataset.enabled !== 'false'; const minAgeDays = parseFloat(ruleEl.querySelector('.min-age')?.value); const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); @@ -657,7 +698,7 @@ document.addEventListener('DOMContentLoaded', async () => { }); const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; const unreadOnly = ruleEl.querySelector('.unread-only')?.checked; - const enabled = ruleEl.querySelector('.rule-enabled')?.checked !== false; + const enabled = ruleEl.dataset.enabled !== 'false'; const minAgeDays = parseFloat(ruleEl.querySelector('.min-age')?.value); const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); diff --git a/resources/svg/circleslash.svg b/resources/svg/circleslash.svg new file mode 100644 index 0000000..3f16f96 --- /dev/null +++ b/resources/svg/circleslash.svg @@ -0,0 +1,4 @@ + + + + From f156a653ef50e6842b9e4327f9d66c4e6cc155ac Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:04:11 -0500 Subject: [PATCH 05/44] Fix rule toggle button style --- options/options.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/options.js b/options/options.js index 68b58da..336c30a 100644 --- a/options/options.js +++ b/options/options.js @@ -336,7 +336,7 @@ document.addEventListener('DOMContentLoaded', async () => { const toggleBtn = document.createElement('button'); toggleBtn.type = 'button'; - toggleBtn.className = 'button is-small rule-toggle'; + toggleBtn.className = 'button is-small is-light rule-toggle'; const toggleIcon = document.createElement('img'); toggleIcon.width = 16; toggleIcon.height = 16; From a56b79dceaa3a3ab17a82a4e1c2cd21483221420 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:17:53 -0500 Subject: [PATCH 06/44] Add condition toggle buttons to rule editor --- options/options.js | 61 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/options/options.js b/options/options.js index 336c30a..3c2df3e 100644 --- a/options/options.js +++ b/options/options.js @@ -282,6 +282,30 @@ document.addEventListener('DOMContentLoaded', async () => { return row; } + function createConditionButton(label, sectionEl, checkbox) { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'button is-small is-light'; + btn.textContent = label; + + let active = checkbox ? checkbox.checked : sectionEl && !sectionEl.classList.contains('is-hidden'); + + function update() { + btn.classList.toggle('is-info', active); + if (sectionEl) sectionEl.classList.toggle('is-hidden', !active); + if (checkbox) checkbox.checked = active; + } + + btn.addEventListener('click', () => { + active = !active; + markDirty(); + update(); + }); + + update(); + return btn; + } + function renderRules(rules = []) { ruleCountEl.textContent = rules.length; rulesContainer.innerHTML = ''; @@ -391,7 +415,7 @@ document.addEventListener('DOMContentLoaded', async () => { addAction.addEventListener('click', () => actionsContainer.appendChild(createActionRow())); const stopLabel = document.createElement('label'); - stopLabel.className = 'checkbox mt-2'; + stopLabel.className = 'checkbox mt-2 is-hidden'; const stopCheck = document.createElement('input'); stopCheck.type = 'checkbox'; stopCheck.className = 'stop-processing'; @@ -400,7 +424,7 @@ document.addEventListener('DOMContentLoaded', async () => { stopLabel.append(' Stop after match'); const unreadLabel = document.createElement('label'); - unreadLabel.className = 'checkbox mt-2 ml-4'; + unreadLabel.className = 'checkbox mt-2 ml-4 is-hidden'; const unreadCheck = document.createElement('input'); unreadCheck.type = 'checkbox'; unreadCheck.className = 'unread-only'; @@ -409,7 +433,7 @@ document.addEventListener('DOMContentLoaded', async () => { unreadLabel.append(' Only apply to unread messages'); const ageBox = document.createElement('div'); - ageBox.className = 'field is-grouped mt-2'; + ageBox.className = 'field is-grouped mt-2 is-hidden'; const minInput = document.createElement('input'); minInput.type = 'number'; minInput.placeholder = 'Min days'; @@ -426,7 +450,7 @@ document.addEventListener('DOMContentLoaded', async () => { ageBox.appendChild(maxInput); const acctBox = document.createElement('div'); - acctBox.className = 'field mt-2'; + acctBox.className = 'field mt-2 is-hidden'; const acctLabel = document.createElement('label'); acctLabel.className = 'label'; acctLabel.textContent = 'Accounts'; @@ -450,7 +474,7 @@ document.addEventListener('DOMContentLoaded', async () => { acctBox.appendChild(acctControl); const folderBox = document.createElement('div'); - folderBox.className = 'field mt-2'; + folderBox.className = 'field mt-2 is-hidden'; const folderLabel = document.createElement('label'); folderLabel.className = 'label'; folderLabel.textContent = 'Folders'; @@ -473,10 +497,37 @@ document.addEventListener('DOMContentLoaded', async () => { folderBox.appendChild(folderLabel); folderBox.appendChild(folderControl); + if (typeof rule.minAgeDays === 'number' || typeof rule.maxAgeDays === 'number') { + ageBox.classList.remove('is-hidden'); + } + if ((rule.accounts || []).length) { + acctBox.classList.remove('is-hidden'); + } + if ((rule.folders || []).length) { + folderBox.classList.remove('is-hidden'); + } + + const condButtons = document.createElement('div'); + condButtons.className = 'field is-grouped is-grouped-multiline mb-2'; + + function addCond(btn) { + const p = document.createElement('p'); + p.className = 'control'; + p.appendChild(btn); + condButtons.appendChild(p); + } + + addCond(createConditionButton('Stop', null, stopCheck)); + addCond(createConditionButton('Unread', null, unreadCheck)); + addCond(createConditionButton('Age', ageBox)); + addCond(createConditionButton('Accounts', acctBox)); + addCond(createConditionButton('Folders', folderBox)); + const body = document.createElement('div'); body.className = 'message-body'; body.appendChild(actionsContainer); body.appendChild(addAction); + body.appendChild(condButtons); body.appendChild(stopLabel); body.appendChild(unreadLabel); body.appendChild(ageBox); From 57860499504344f0bbb9b0abc2b27c1a04848957 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:32:25 -0500 Subject: [PATCH 07/44] Updated icons --- resources/img/circleslash-dark-16.png | Bin 0 -> 392 bytes resources/img/circleslash-dark-32.png | Bin 0 -> 768 bytes resources/img/circleslash-dark-64.png | Bin 0 -> 1620 bytes resources/img/circleslash-light-16.png | Bin 0 -> 393 bytes resources/img/circleslash-light-32.png | Bin 0 -> 764 bytes resources/img/circleslash-light-64.png | Bin 0 -> 1572 bytes resources/img/plus-dark-16.png | Bin 0 -> 199 bytes resources/img/plus-dark-32.png | Bin 0 -> 301 bytes resources/img/plus-dark-64.png | Bin 0 -> 431 bytes resources/img/plus-light-16.png | Bin 0 -> 215 bytes resources/img/plus-light-32.png | Bin 0 -> 305 bytes resources/img/plus-light-64.png | Bin 0 -> 457 bytes resources/svg/circledot.svg | 4 ++++ 13 files changed, 4 insertions(+) create mode 100644 resources/img/circleslash-dark-16.png create mode 100644 resources/img/circleslash-dark-32.png create mode 100644 resources/img/circleslash-dark-64.png create mode 100644 resources/img/circleslash-light-16.png create mode 100644 resources/img/circleslash-light-32.png create mode 100644 resources/img/circleslash-light-64.png create mode 100644 resources/img/plus-dark-16.png create mode 100644 resources/img/plus-dark-32.png create mode 100644 resources/img/plus-dark-64.png create mode 100644 resources/img/plus-light-16.png create mode 100644 resources/img/plus-light-32.png create mode 100644 resources/img/plus-light-64.png create mode 100644 resources/svg/circledot.svg diff --git a/resources/img/circleslash-dark-16.png b/resources/img/circleslash-dark-16.png new file mode 100644 index 0000000000000000000000000000000000000000..a7057deb4d2f26e80f2726ca830bb866b0621565 GIT binary patch literal 392 zcmV;30eAk1P)Vw@yjI4lW9DSICHG@mc=*buB>(wcb|xVe)F zdV#&X>8b;KoVBx#ZGB@museld8n_`CV<)iOF+!K@GPe@5)Vvtxp8o?5T;ZYI!jak} z{eOVhbB+5YL4|=j`g6d~xYjaKsO>pOC6_s8Q`=72U%1g>N mmeCzc5wV`@j0000v2QVT*%Acsn z8{my5AVQ*Eh~v=ffc~fS=I|MmbWzeNNoW1s zrlcK7Ym&ZaiUzczeLxvl0`_Pudq4%4@;*&tH#6u3CV?*v0Jps+$tG|Y=*dfmM5#WfiE%~`=T$Zf722N%rZ~^#eg1iGRrR^lrap1l2Uk8R;CD3oc_km|XF#~we zA}|w?IFyN`ZywUV4@~3$wprMvUgn6-MC>$ajGuN0d>43O;?`{{F12vl-;yuwN#Nr? z)~cV|#FVIYEE|?t4qP+vuy)zRe~?5Vl+R)pz=JOP^-4@&0I2%Kat`n@;HUTh4P0$t z_sHAq0fUyn4U?XcF6=z$sKL1gsOYbRAwTX*`kdh^8I$x<(lJSYB;As*QesQVIQcKenA7dfR z*~KYSqZXi3;8#ri`$+`)a2Ex*W}?_Z@E%~z$JqdmCJ_*>hYK>(A#efDP24+nEs5zI z_0lf*Jriq|CJl*jZPR*`iM%bD-T69jBDQPfJRFf&#ogyw4%~+byyd3UW+MI^%eX6b z6xWIe+mfGqOs&*{%qH!;6$0T*Oi-(iSa!T7$p&znHXbHc`*3a40_jh)D)0tZ=Le;h yrUl6SIv$XFoSs(w<8)i@arz@musj4hD*pgE9+=@hjn3Tw0000s3V zK~#90<(tcoR7DiVf8z`cQ3e@CGYINJ5s*j_1ZRu}o45Z#FQnjtP=fJig~0Tl$4M|aQXqIyD2b=B?L_tCeFUs8*zTXpKZs_v~* z=e9}W|BRc8W}s*kt$^{sEk!cW(S$iB0SkeCUfa0oa7To3de3rqw0fW^RaptlbF6Tox8_p!KkkMJtlFOL|&T zdkPorsM6S8r5VCiNiQdvqtO9vlHQD9WLVM%lBTB+I#gyz+ESe(g!OqUpiR`DcT4&%Amo6g+a$f?c~+A2U=js<;UV^;n!DS_vHU>wIWljD!=667 z8&yD`he$=udCw=YyxspX`9GGtyoVAxec0(w=u_RGrW^EOM% z+hwL|&kiL?lij|)NX;s0kEA<*p+u*xWat4701tVd4FXGn-y_B!0d_iV9l-N7Jw4j< zY5l1tT0W`UyJdxE{6V*$J7|)}bxzA1OaDv4BWaAJvkr~2q=|MvcL_XBZ2)r*Ci;>$ zx`Exm9M7{sU>Wd71N2J3UZ<@cSY$W2!uELx9cW$fYe0%mvy97-^8 zBj^G45w`qN8ne8OZ#-?IWUQxvGH^WMUU$VtJ0vE8=i|!;4Y`_H@o60;D}T20Pan}K#`UQxR2UP+fk=lR{=*oM0}se zaggU1UR?NiA7x}$0lrVi#R(=Z((P?5@w6QrRdbGz-Jav{nFf!;SsrQAl+bbb)~Ms~ zc}z}@jb6+>eH^rc+mGN$;C83&Ti`Fd!S^Aw&C~W`j0sywc<}RFE1So4d_mEQq*)Q} zmL`CiN4Z}ygQ6hx%#i7y2Wa}f8uvs_W4$Uaz6rHoUHZHA3C*B+}>0z07aNfc%l>b#Xf=!LkOTxgq zh{=Z~eJFE3pS9q#WWMmCQf1dRN;fGps};R5R%AXsS18@h@)$|aNcvRK$CSuZ3`Z0I& zGl4s*?QPXQ!rbDX!i46BfUkisabkE`qZtZ_!pLv4e;36-(F!OWMgKfqIQ|8rXRcaJ SCJyfa0000R>Cj?>Xm>#>AZS??OT!&&c?Wl`%pOX%16>*YJQZTwte& z?cx$++@m!|5S1VCg3Yo*P2nKa+*eAfz-d%o20p_yNG3Qb37&DW0{ATUR~S|XY+zR2 zc^SCq$qXGhj83Npa6e+_4Q_qoK>QX6{sI>SQ|!Zf{R&O8YXsfEw5x*=E>{6Z&NZG& z7kV)ygB8HrVdGJKknUng2LFIxaWgka5o`s?IDGCFvAvKz!R^8**SdRxVKk|FOfx*; nxb9Y3@|@ct8QyZvpG|o`f`EVKz>1ZM00000NkvXXu0mjfAl#rS literal 0 HcmV?d00001 diff --git a/resources/img/circleslash-light-32.png b/resources/img/circleslash-light-32.png new file mode 100644 index 0000000000000000000000000000000000000000..a5cddafe898238f4751f222f4e8f2f968090df48 GIT binary patch literal 764 zcmVz$qXR{S;^b3qUCqb^`G*FbjN)AUxC_HntDU07HEV04@Nl z9%vJI1dIV^jBQcBEozpm&ud^LlK|ib@WFs=0k_hmRQ;>KJLA6#jHVKxk+;cz1J8j% zT5^L^0G2!vJ(&ob^HoD$%S!BJ;l!Jc6@N|mPP4}NSx)39z)OQ$3l*TH)?V$K{iHpD zysFc&ZK|UbCZKLqre1JK0rGN@(JLp`hN#* zN7y~4vtw4QfO{rAV|hnzY{w1H9V?)wAAezXxV$1a_C+mk0j5QS0ztd1&tnm3k)k>0 zCV*GKIp7CyUql*lJ|glHIM&xIz+VCFLxCF6-5-se`9M)u&piCfFEy0C`}urw6;b93dBSruSQLbQIZ;cFfs92f#{pD zfvV3cl9C9bPly$bzED4+Eg_MjgeFEys`4$Mx3}ZN%!TdTvwP?6-MxDoPBIU(JA3AT z{%7XQ?0hJ#H8--{lr?~?B~t>FQcX&!TQbFvb{9}eEdka61HcNP16T&MIz`U`r-9?Z zBrpbiqqRPhD!w!bP)gkiJOw-ftR{X-n*)9XMt~8m^>i`_1ERG~EMdI-DKHD@gwrf= z0O(H?b3%r%1`elSbXXb%RwaSGqAsA6Y5`scUIgkqKbr-91I_>!?XzZ}9k?5)FGJ5e zz^hv83%;0(qyn$^0w*c$t!dzW;2B^g&=8@s0q6vt1wOz8#Zvlz1=dux3ji+w|3-ky z1N(r@KwSkS*IOO?tj@6dt^&`eP5_{QHzI({S#K{-qJKJduK;%2I>PGTks1Nk_;({J z9|s;t+He8KgTO=tj^VTj2=n|LFcj6dQy`%c*cCy-_CyK5MES73_jt7ocQCaBry@ug zOoRZSFC^ey05)W3{CI@%1?TisL;%o&Jk(vSapMc=7ofQk0(OS<`7E1#nDJ8~&)6ueifo)^B`Ba}bjMF6l===-yzFuhIE^Uk*&6#qNq zfMq@eydqTi(L(UNqvK=Y+n0R^kXV0oAsKJ=JSKcU7D)iGL})tGK>b28-s-v;m=U^i zYp|TVSosje3z(m4t!GJDlu~WLAz+p8dIH#_wf@^zpVm5uSvm_!9k4$5z%{mC(DjvX zrEx_m)o$xqBfOpfHfpW^ilO>T;n_ei;}Wn!_~5%36?=;{{xIR5-&l_49r;#xHs404 z-~3GR(2?5Aj|zr5PyG!PFVP>13WDdI!Y3C(z$;PDDM$BNF$gN96wu@}m$N?YtXxSt z;FXYbPQE#q3C4iM6vu>134ypK*7#Av1Ky^X0WVD4J%c)>)Z#*>s=LD(7++F?1YE}a zMFqv}60s#i<6E43UTb~1khw;HHW9;Z@@+{VNO7F^5qBH@;=Eki!yqXSbtURf)Z(ZTvAe=;rWV$eV&tD0;H+q!{ydm<#9er zHVDtg=H-O`>i!O&ri|ZVsx-s%z!CO4eCxb&!iV1rz&9@><1Lmg0)``QIcXxaS9tcE z@NAycNS_aigt^!@fjMbWG!Jyv;!qcYXC-kLMXds9S$ub`H~Rpl5dq77v22Kf`vnDa z!1`MDyu}o@G1BKvfK@aqD475nYuNLPfMWulgQ4FGAkaAsPrhXwhG3t zLmui1X;)eLVgT44QTZJ3A<&V6;hmTnUer-pZ_mdjz=9bp5g7Bpe#{ni?44!IFYOl0 zUVqeK=_|f{J`n;exE{uE>5;$j-?C$ z%P~jT?!fHWC|gWpc5(l}1ho^u5#VdB_4!osrA W9v{7)Jvxj40000N&PEETD8$pnF~p)b`Op9V_ROjcoeg`m zMIN*0u<&p+UfO-Kl+n?EhbN)MZLv19(t!z$x>M|WT=RIDnGff!+Q}e3gLCQSLQRPu o4l)(Y98X(KTX+;ixLFt(1S5r_Qda5p0nKOdboFyt=akR{08x)R?EnA( literal 0 HcmV?d00001 diff --git a/resources/img/plus-dark-32.png b/resources/img/plus-dark-32.png new file mode 100644 index 0000000000000000000000000000000000000000..fdb1a30bedc11e42ccd5fecf2b7f7b04d8d3c2b4 GIT binary patch literal 301 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdz&H|6fVg?4`^&rfc{H*UVP*AeO zHKHUqKdq!Zu_%?Hyu4g5GcUV1Ik6yBFTW^#_B$IXprT!#E{-7)t#7Aqi5FP6;Cv18{sz`&F6o1s!Ouko?Z{m=Zr=g&JJp{|#C zocBY5*4pe>zL%`8I6v6L?wQH4?%+O#zdrld3p}wjSk}$@s&V!Ke|r|T1O^~7m7m z!eoYR4ZhzQuRXf+{lMuTe>msxaB3f5-oS45_`UK5*1C1NNxF@)_4V^UR;G7EFwRTs zGhP1M*MKpHaW=~a)*PV&VjDaeq7SHH6?n&drSItZ=imBypViK@{3c%$yt6z)G_QU6 z^KaYzKHging=HC(_1kvAZ`A-UhPUPiub;eOU65fAUMjq1FRR}Hcjkr0ml@a}tbJG` z{<4nW1E*!E&RJ5qALP_BoqLmx))*zTHni55q_Rb@++@jxxV(N&PEETD8hkw+Op&3M4h*%;@(a^_Q^fX>(nSv;4*%86G* z7$<}<+g@m|h-O&8`YmybDnr;rW{U=93kOm4hcWgk*W?~Y1m&HdJ(KtRnKK;WMZ7%? ze=B<&Y`^N}H9lfL$-{k4(T*`TEjGz<6Z@efAHqI4>dz}rD?7>J5psu#y?}wepyAc` xN$Ynpeb<<$s{3<}-~CNbs_qIU@QU7N=J)pu8FoxUvb>SI#(m^ELi+u`80pbhgnYLz3s!n(?AChcDu(RU`~ z^qCAsDfR`d8MF8coO5Law@;e(^mTTcoB;nLJpsYFOUgaJr)H|ZG&KGmygOj}0jK-w z>pxXGb_6iK(~DW{_u2UZBUHakgV=&XhNy;P*aT)hW9G3F|M7AE-#v%+@z-Vx1}19P zX1&?zoc7O3rF7D^NuTD+M0>`*-^s@k5!?7BUViny;@mnmlX^}bJLx$I9P7?so2Qzq z^~ThSh5bF#Q+9pPmXKwCD;s>%?5^I(?|GKqzB=_f+62(uiEFM>(==4eL6Sc$`9}0>n-gH5UpngYnB_iefH6L0*nR*Pgg&ebxsLQ E09huj2mk;8 literal 0 HcmV?d00001 diff --git a/resources/svg/circledot.svg b/resources/svg/circledot.svg new file mode 100644 index 0000000..e380e09 --- /dev/null +++ b/resources/svg/circledot.svg @@ -0,0 +1,4 @@ + + + + From 20dfba840689221271cd0ac1320a1819479bf926 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:40:03 -0500 Subject: [PATCH 08/44] Add active state icons for rule condition buttons --- README.md | 5 +++-- options/options.js | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7637f85..7436436 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,9 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e reorder them, check *Only apply to unread messages* to skip read mail, set optional minimum or maximum message age limits, select the accounts or folders a rule should apply to. Use the - slashed-circle/check button to disable or re-enable a rule, and - check *Stop after match* to halt further processing. Forward and reply actions + slashed-circle/check button to disable or re-enable a rule. The small + circle buttons for optional conditions show a filled dot when active, and + check *Stop after match* to halt further processing. Forward and reply actions open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the configured rules. diff --git a/options/options.js b/options/options.js index 3c2df3e..1cfd03a 100644 --- a/options/options.js +++ b/options/options.js @@ -286,12 +286,18 @@ document.addEventListener('DOMContentLoaded', async () => { const btn = document.createElement('button'); btn.type = 'button'; btn.className = 'button is-small is-light'; - btn.textContent = label; + const icon = document.createElement('img'); + icon.width = 16; + icon.height = 16; + icon.className = 'mr-1'; + btn.appendChild(icon); + btn.append(label); let active = checkbox ? checkbox.checked : sectionEl && !sectionEl.classList.contains('is-hidden'); function update() { - btn.classList.toggle('is-info', active); + btn.classList.toggle('is-active', active); + icon.src = browser.runtime.getURL(`resources/svg/${active ? 'circledot' : 'circle'}.svg`); if (sectionEl) sectionEl.classList.toggle('is-hidden', !active); if (checkbox) checkbox.checked = active; } From 1432690fa818bde2ddf0db02cf5daec5bb3d18be Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:55:44 -0500 Subject: [PATCH 09/44] Fix rule condition toggles and add divider --- options/options.js | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/options/options.js b/options/options.js index 1cfd03a..096f1a8 100644 --- a/options/options.js +++ b/options/options.js @@ -282,7 +282,7 @@ document.addEventListener('DOMContentLoaded', async () => { return row; } - function createConditionButton(label, sectionEl, checkbox) { + function createConditionButton(label, sectionEl, checkbox, clearFn) { const btn = document.createElement('button'); btn.type = 'button'; btn.className = 'button is-small is-light'; @@ -300,6 +300,9 @@ document.addEventListener('DOMContentLoaded', async () => { icon.src = browser.runtime.getURL(`resources/svg/${active ? 'circledot' : 'circle'}.svg`); if (sectionEl) sectionEl.classList.toggle('is-hidden', !active); if (checkbox) checkbox.checked = active; + if (!active && typeof clearFn === 'function') { + clearFn(); + } } btn.addEventListener('click', () => { @@ -523,16 +526,30 @@ document.addEventListener('DOMContentLoaded', async () => { condButtons.appendChild(p); } - addCond(createConditionButton('Stop', null, stopCheck)); - addCond(createConditionButton('Unread', null, unreadCheck)); - addCond(createConditionButton('Age', ageBox)); - addCond(createConditionButton('Accounts', acctBox)); - addCond(createConditionButton('Folders', folderBox)); + addCond(createConditionButton('Stop', null, stopCheck, () => { + stopCheck.checked = false; + })); + addCond(createConditionButton('Unread', null, unreadCheck, () => { + unreadCheck.checked = false; + })); + addCond(createConditionButton('Age', ageBox, null, () => { + minInput.value = ''; + maxInput.value = ''; + })); + addCond(createConditionButton('Accounts', acctBox, null, () => { + for (const opt of acctSel.options) opt.selected = false; + })); + addCond(createConditionButton('Folders', folderBox, null, () => { + for (const opt of folderSel.options) opt.selected = false; + })); const body = document.createElement('div'); body.className = 'message-body'; body.appendChild(actionsContainer); body.appendChild(addAction); + const condDivider = document.createElement('hr'); + condDivider.className = 'mt-3 mb-2'; + body.appendChild(condDivider); body.appendChild(condButtons); body.appendChild(stopLabel); body.appendChild(unreadLabel); @@ -582,7 +599,9 @@ document.addEventListener('DOMContentLoaded', async () => { const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, unreadOnly, stopProcessing, enabled }; + const rule = { criterion, actions, enabled }; + if (unreadOnly) rule.unreadOnly = true; + if (stopProcessing) rule.stopProcessing = true; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; if (accounts.length) rule.accounts = accounts; @@ -760,7 +779,9 @@ document.addEventListener('DOMContentLoaded', async () => { const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, unreadOnly, stopProcessing, enabled }; + const rule = { criterion, actions, enabled }; + if (unreadOnly) rule.unreadOnly = true; + if (stopProcessing) rule.stopProcessing = true; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; if (accounts.length) rule.accounts = accounts; From 32c8a0f4158261ac0035fd83352f9e2deaae6904 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 02:22:44 -0500 Subject: [PATCH 10/44] Update AGENTS.md Adding direction to skip running the build scripts. --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 9b94461..401f962 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ This file provides guidelines for codex agents contributing to the Sortana proje ## Testing -There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. +There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. Do **not** run the `ps1` build script or the SVG processing script. ## Documentation From 598e74f8cb342348b1367cdb22d158e49aa606c8 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 19:18:21 -0500 Subject: [PATCH 11/44] Add notifications permission --- README.md | 1 + manifest.json | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7436436..3853127 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Sortana requests the following Thunderbird permissions: - `accountsRead` – list accounts and folders for move or copy actions. - `menus` – add context menu commands. - `tabs` – open new tabs and query the active tab. +- `notifications` – display error notifications. - `compose` – create reply and forward compose windows for matching rules. ## Thunderbird Add-on Store Disclosures diff --git a/manifest.json b/manifest.json index fe019ab..c3b45a7 100644 --- a/manifest.json +++ b/manifest.json @@ -32,17 +32,18 @@ "page": "options/options.html", "open_in_tab": true }, - "permissions": [ - "storage", - "messagesRead", - "messagesMove", - "messagesUpdate", - "messagesTagsList", - "accountsRead", - "menus", - "scripting", - "tabs", - "theme", - "compose" - ] + "permissions": [ + "storage", + "messagesRead", + "messagesMove", + "messagesUpdate", + "messagesTagsList", + "accountsRead", + "menus", + "notifications", + "scripting", + "tabs", + "theme", + "compose" + ] } From d3b16640b99afcfc6b16027de5448551398ea64d Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 19:43:24 -0500 Subject: [PATCH 12/44] Handle endpoint changes without restart --- background.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/background.js b/background.js index 06683e2..c7579d0 100644 --- a/background.js +++ b/background.js @@ -417,6 +417,20 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } + if (changes.endpoint || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { + const config = {}; + if (changes.endpoint) config.endpoint = changes.endpoint.newValue; + if (changes.templateName) config.templateName = changes.templateName.newValue; + if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; + if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; + if (changes.aiParams) config.aiParams = changes.aiParams.newValue; + if (changes.debugLogging) { + config.debugLogging = changes.debugLogging.newValue === true; + logger.setDebug(config.debugLogging); + } + await AiClassifier.setConfig(config); + logger.aiLog("AiClassifier config updated from storage change", { debug: true }, config); + } if (changes.htmlToMarkdown) { htmlToMarkdown = changes.htmlToMarkdown.newValue === true; logger.aiLog("htmlToMarkdown updated from storage change", { debug: true }, htmlToMarkdown); From f56057c0428c5f5244b6e1de092d7ae12356f008 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 20:56:55 -0500 Subject: [PATCH 13/44] updated manifest to new version for bug fixes. --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index c3b45a7..a3c9f7c 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.1.1", + "version": "2.1.2", "default_locale": "en-US", "applications": { "gecko": { From 183ca8f355783f43a21555338cae6144190c871e Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 17:31:03 -0500 Subject: [PATCH 14/44] Add token reduction options and context length cap --- _locales/en-US/messages.json | 4 ++- background.js | 64 ++++++++++++++++++++++++++++++------ options/options.html | 11 +++++++ options/options.js | 12 ++++++- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index 983bb86..fb439b9 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -18,7 +18,9 @@ "options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" }, "options.stripUrlParams": { "message": "Remove URL tracking parameters" }, "options.altTextImages": { "message": "Replace images with alt text" }, - "options.collapseWhitespace": { "message": "Collapse long whitespace" } + "options.collapseWhitespace": { "message": "Collapse long whitespace" }, + "options.tokenReduction": { "message": "Aggressive token reduction" }, + "options.contextLength": { "message": "Context length" } ,"action.read": { "message": "read" } ,"action.flag": { "message": "flag" } ,"action.copy": { "message": "copy" } diff --git a/background.js b/background.js index c7579d0..c44bfed 100644 --- a/background.js +++ b/background.js @@ -26,6 +26,8 @@ let htmlToMarkdown = false; let stripUrlParams = false; let altTextImages = false; let collapseWhitespace = false; +let tokenReduction = false; +let contextLength = 16384; let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; @@ -125,12 +127,16 @@ function byteSize(str) { } function replaceInlineBase64(text) { - return text.replace(/[A-Za-z0-9+/]{100,}={0,2}/g, - m => `[base64: ${byteSize(m)} bytes]`); + return text.replace(/(?:data:[^;]+;base64,)?[A-Za-z0-9+/=\r\n]{100,}/g, + m => tokenReduction ? '__BASE64__' : `[base64: ${byteSize(m)} bytes]`); } function sanitizeString(text) { let t = String(text); + if (tokenReduction) { + t = t.replace(//gs, '') + .replace(/url\([^\)]*\)/gi, 'url(__IMG__)'); + } if (stripUrlParams) { t = t.replace(/https?:\/\/[^\s)]+/g, m => { const idx = m.indexOf('?'); @@ -138,7 +144,7 @@ function sanitizeString(text) { }); } if (collapseWhitespace) { - t = t.replace(/[ \t\u00A0]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n'); + t = t.replace(/[\u200B-\u200D\u2060\s]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n'); } return t; } @@ -157,12 +163,26 @@ function collectText(part, bodyParts, attachments) { attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`); } else if (ct.startsWith("text/html")) { const doc = new DOMParser().parseFromString(body, 'text/html'); - if (altTextImages) { - doc.querySelectorAll('img').forEach(img => { - const alt = img.getAttribute('alt') || ''; - img.replaceWith(doc.createTextNode(alt)); + if (tokenReduction) { + doc.querySelectorAll('script,style').forEach(el => el.remove()); + const walker = doc.createTreeWalker(doc, NodeFilter.SHOW_COMMENT); + let node; + while ((node = walker.nextNode())) { + node.parentNode.removeChild(node); + } + doc.querySelectorAll('*').forEach(el => { + for (const attr of Array.from(el.attributes)) { + if (!['href','src','alt'].includes(attr.name)) { + el.removeAttribute(attr.name); + } + } }); } + doc.querySelectorAll('img').forEach(img => { + const alt = img.getAttribute('alt') || ''; + const text = altTextImages ? alt : '__IMG__'; + img.replaceWith(doc.createTextNode(text)); + }); if (stripUrlParams) { doc.querySelectorAll('[href]').forEach(a => { const href = a.getAttribute('href'); @@ -198,7 +218,15 @@ function buildEmailText(full) { .join('\n'); const attachInfo = `Attachments: ${attachments.length}` + (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : ""); - const combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); + let combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); + if (tokenReduction) { + const seen = new Set(); + combined = combined.split('\n').filter(l => { + if (seen.has(l)) return false; + seen.add(l); + return true; + }).join('\n'); + } return sanitizeString(combined); } @@ -233,7 +261,13 @@ async function processMessage(id) { updateActionIcon(); try { const full = await messenger.messages.getFull(id); - const text = buildEmailText(full); + let text = buildEmailText(full); + if (tokenReduction && contextLength > 0) { + const limit = Math.floor(contextLength * 0.9); + if (text.length > limit) { + text = text.slice(0, limit); + } + } let hdr; let currentTags = []; let alreadyRead = false; @@ -391,7 +425,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "aiRules", "theme", "errorPending"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "contextLength", "aiRules", "theme", "errorPending"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -401,6 +435,8 @@ async function clearCacheForMessages(idsInput) { stripUrlParams = store.stripUrlParams === true; altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; + tokenReduction = store.tokenReduction === true; + contextLength = parseInt(store.contextLength) || contextLength; errorPending = store.errorPending === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -447,6 +483,14 @@ async function clearCacheForMessages(idsInput) { collapseWhitespace = changes.collapseWhitespace.newValue === true; logger.aiLog("collapseWhitespace updated from storage change", { debug: true }, collapseWhitespace); } + if (changes.tokenReduction) { + tokenReduction = changes.tokenReduction.newValue === true; + logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction); + } + if (changes.contextLength) { + contextLength = parseInt(changes.contextLength.newValue) || contextLength; + logger.aiLog("contextLength updated from storage change", { debug: true }, contextLength); + } if (changes.errorPending) { errorPending = changes.errorPending.newValue === true; updateActionIcon(); diff --git a/options/options.html b/options/options.html index 57f407c..108d7f9 100644 --- a/options/options.html +++ b/options/options.html @@ -144,6 +144,17 @@ Collapse long whitespace +
+ +
+
+ +
+ +
+
diff --git a/options/options.js b/options/options.js index 096f1a8..f5af1b6 100644 --- a/options/options.js +++ b/options/options.js @@ -16,6 +16,8 @@ document.addEventListener('DOMContentLoaded', async () => { 'stripUrlParams', 'altTextImages', 'collapseWhitespace', + 'tokenReduction', + 'contextLength', 'aiRules', 'aiCache', 'theme' @@ -115,6 +117,12 @@ document.addEventListener('DOMContentLoaded', async () => { const collapseWhitespaceToggle = document.getElementById('collapse-whitespace'); collapseWhitespaceToggle.checked = defaults.collapseWhitespace === true; + const tokenReductionToggle = document.getElementById('token-reduction'); + tokenReductionToggle.checked = defaults.tokenReduction === true; + + const contextLengthInput = document.getElementById('context-length'); + contextLengthInput.value = defaults.contextLength || 16384; + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { const el = document.getElementById(key); @@ -791,8 +799,10 @@ document.addEventListener('DOMContentLoaded', async () => { const stripUrlParams = stripUrlToggle.checked; const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; + const tokenReduction = tokenReductionToggle.checked; + const contextLength = parseInt(contextLengthInput.value) || 0; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, aiRules: rules, theme }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, contextLength, aiRules: rules, theme }); await applyTheme(theme); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); From 91b8d1557322d1fee91b92ed9d3fd0a2387a96f7 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 17:33:59 -0500 Subject: [PATCH 15/44] updated icons --- resources/img/circledot-dark-16.png | Bin 0 -> 413 bytes resources/img/circledot-dark-32.png | Bin 0 -> 828 bytes resources/img/circledot-dark-64.png | Bin 0 -> 1753 bytes resources/img/circledot-light-16.png | Bin 0 -> 408 bytes resources/img/circledot-light-32.png | Bin 0 -> 813 bytes resources/img/circledot-light-64.png | Bin 0 -> 1737 bytes 6 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 resources/img/circledot-dark-16.png create mode 100644 resources/img/circledot-dark-32.png create mode 100644 resources/img/circledot-dark-64.png create mode 100644 resources/img/circledot-light-16.png create mode 100644 resources/img/circledot-light-32.png create mode 100644 resources/img/circledot-light-64.png diff --git a/resources/img/circledot-dark-16.png b/resources/img/circledot-dark-16.png new file mode 100644 index 0000000000000000000000000000000000000000..db602abfd60d10c744105a8849186b1ed8d39651 GIT binary patch literal 413 zcmV;O0b>4%P)13?T$|D*yAkU+^Ufr|1vMlM2tk2Fi$p_Gg=Qbc zdccf!Lo9u1wD$LWum1!k4aO$>KXODLxQ`9ifFhY$ zKnHj<>?OIjaasY`k#s9*Ns`~`lcXC-4M_)*wp}hrnn~(364IXy8@a9~NGb%Z){L+Z zJQ#M9+@^7w@x#dWyy#Cn2k!h9dS;Sr0dj6@JWP!=Fq6z{k^?WmMQS8b+H+t^`Zj0G zZJF#Ta7l%gy^^k(q~P~46*lt`$zv*IR7X=%Zs%7?A5}%)+lquESY;#500000NkvXX Hu0mjfLU^6T literal 0 HcmV?d00001 diff --git a/resources/img/circledot-dark-32.png b/resources/img/circledot-dark-32.png new file mode 100644 index 0000000000000000000000000000000000000000..d29ac166e645083f94860aed3be1122ef452ae70 GIT binary patch literal 828 zcmV-C1H=4@P)==>Ghs4Wr& z1QkU80BE8Kt3skC#!fc-Sj>zZhTZ+XU6=SKC$pLFT;4f*=FCh6(EmhZ4L0qP_Db3+ zX@{T7OIngNE9rYx-hedH1PlQ$fCA-F0LFof-X_s^l0qYJ7WiBy;G&mO$OD5wL#+fh z1J5F-6Tl^)8`xlUbAIkJE=v|qf!1mXbOH0m$!nme(n=!f1*VPt9MGATKx-8IK9H>< zc#sTm6DS%IYcf&lKM!dy0s}P!j%Co07$vHSSoxe;`K>wx4|3P|UA3rqtA%QRRb$%A zNfwy#zVnzZYL$iz%W4WPt@IiH7fKNb+h<}Of(M!OF~-AO);Gj9N&Efp$0-_Ui=<&m zFTLE8)SSZZi649ATKeA%95LzXPFTez-~(&%9%xCh-DUI-*$i~}u_)h!t~ zyc8vMNxGI``%zLjemkU<({Q^Vze!q7aB|R(4<()QzcZ3fO6pIr{q19H2?Rn00;^1d z0}HF0Evr@D4rwI=U;WrFDd(A%!?>hPxB3 z#3-vV30%RQYB{5u^K(~O*=hkJxD&KwoHW%tz{>83vE-!`7H|bu^TWhS6Yd%{M&+m3 zGVmNYkJ)OKBm-5yj@#rOr`xppak?n?IQ>zLvswt$6@LJo{<`ZPn7_sV0000fdnWqBdN_-H z*7~n?_SyUFwfCuz#Qz!H(k75L(oR4h;QBNPRF=WV0AK=82aEtl0)v5mR=ab+Y2XNO z6xaiN1N>D=aup>S18xT90<(cJ6yMW2fFFSM8EN#wE(l3JrB2djNp13vHc6W#)%xV@ z%g`7}UzZ^0Fl>`_w-4?~UxB{BYruS!N+}=sM)4-R&c3>}X0=N>=wkI$Qs0AhgQ-DF8*gpZx0=|#M+D4IG=UGWD z5nZ%h(nd*BC3R2GMJqElHfC℘^*1#rP=nh6+i`B8YTIS|e$Af}q1>grs#DA7NNp z$^=wMdN;!Peo6P0AaGmmmvkV4$LdlgV0nbo_a*f#L-1MYA?Xc;_l70D6EHu-QM06( zWenVwISL<(=J`s%_z)-OB~4B-_^do6smZ~wS<={~3Fs^7M2PXO5PUERu7`A7QgxC9 ztaKRdP`cedOy)SaO43`!CZI-AtHa=mE)Z;(yzSuID(RkL67ZG7*pDjjXkVyQNjm7@ zy|vH;)H;l`t3303Qh7k`>CwqOY@PnHL$?mtUDWq>dGRf#c3|TrS7X;=Xb^Cc+*MgS zFbX*4fnYf>5;#n5A*Kx&idmdpg{ueDlfNY0#D6DvFlDvPmaY<*V>fgqV75c|fd^9M zhKutoTVcNf1Aw#SCgwDz>$G_y)*E;Pco0(n0rSf31hxQ|Jo)JXoC0nP=-PoBFa=lx zYgC-!7(Lf`9HF0-)adY$MWdv8kN!S#_;@sG7N~V(a)+lU@;a~vQxXruU|=n#%;p2z z9lE;Ug`I#AjvMxRATk$NP(b^Iz)TN*zjNqvVzkGApD|NN7HycFo3I*Q98>|nV)hCO z!6{6=F^TV(H;Zcp&S9s~LVf0CP+Gni$oujOzr%yM`l*XH>K z%%W>6`Co*^B5XN45cub+#<7Il=U75c^|(YWmrQrWvD-;BDiGo#g#esLIWkvWu*cFRCyT{# zZ58Txb%fJ{GIMaLgP$PxRF}o7LhKboU`d4Y4oM$M8d-wCM@d?*=%ccW3$AYF_oTSowPfPIWE5gQ*b#Bw1l*+ z!j!RUF~y9B$z9L8a168Wzdsgh3rT;Ns0LoeRCVM2=+KTSRGj5n6}k^5&>_VC@uC%2 zmGNKv22O(R8I&4HUqp0K-`FB)v=8pS2deQh@2l&8Wm2@tj4R`OaxT?xbAvDkl4fIm zt%=*MZL|aXFqNG3z#m@N1yHI41kn#u0IdUtV>VuH%ILHEfx_(K{*EclAH$qU+(oc} vSpt+J0a3{QHhVEG0%<28b)^0Cbn5sQ=z)1={^||<00000NkvXXu0mjf@q#fR literal 0 HcmV?d00001 diff --git a/resources/img/circledot-light-16.png b/resources/img/circledot-light-16.png new file mode 100644 index 0000000000000000000000000000000000000000..4809c5ac4d11094ddd47913c943a9b612fa3dfb2 GIT binary patch literal 408 zcmV;J0cZY+P)adeJhfOC9u$7hjl3OCLSg!>xvHcsl64}9dW575*>N_W} zjO&cW7B-!n(SmuLVlXOCODQ)v!2nhwVy?mp-mpB77WH&E$^CLlc44f~6_#+7vCzV1 zk~7jAT%h}pN8H6(?ZQS5N#_sb$+U5`JxB*RBz;Aae1|>tASF1IA}{IlOoh#6+ppNy z9_7|Dt>Zi&sad8EoMBCCnaU6AJd!QLeMCI<%KHSW>~1$Tknv>z0000$s4rmL2(nZ6CYrb+pe}$T=)$;jVTiKxDfAJP zC>Qa9yn$e%38;{u55OU(TyzZ-!TV|@{Lj&v*TFg01zZ4* z1E=-%DzFYLipcL=fFhD7I80Y-sJ5OxM}D{u$+(S&eKEop8Q7y?@M zB>*@IyorI%0{4I(;E-8c)^}qXCF}DV=*%SmI0GyhkdMHnkdUf>FYw9ue*wCK1OS~e z^5;M)l-%Gr;4#oJBKAZg>7PeoZvcaNi4%kj84*+QoQN;JYrg!+zmgl=rv`U0QGk|O zl&8Q4cxEvz-4E{)xfT(uir0LLN$Q3N7Y zE`!_-ySCJHRdc`X&Ij;IpIy$mvWRS^0L%avfgAqAalmym*PPk8GCtt}etH=`Ij*vk z9A^V6V41y1xI{)J0OyG%VNw~61(Z`8zP-+?O#nv%{2nMr9jL||(X39+9bhAxSB~kV zPB?19(|Z8q{RV$HNk9d^Q3(DpQRHUS0xT-_3UGLj09|<7sGKNJz@)()*fp1+bc$jR zB=)0Tk#-DDrLjm%84$tm=i~TYt!&no z_1%3_YBua0O?d*cchFQv+}72S=2n1#_&?7jzzC@TBfvE2SF<|s78tgrmS#=~f`@ai riocwmVE5&84SzZPlM64efTG7=cGHKb4{C8?00000NkvXXu0mjf>c(M) literal 0 HcmV?d00001 diff --git a/resources/img/circledot-light-64.png b/resources/img/circledot-light-64.png new file mode 100644 index 0000000000000000000000000000000000000000..cb8634a6126b789ca3b9b97553f9b376bd851849 GIT binary patch literal 1737 zcmV;)1~&PLP)n^aOAi z*ah?gKLJ0A$ni|^WnloN)Nr5+m;+28c~9#Heg)P5YeeLb8-zX*5oZPM;sMir@gvnU>GnGsIP+0 zd%yw_ITaUkfKqsUBCv@mQ{o{TX0i!(ETr~La7 z!hZwqb*;Ds$9=%|2plW2GQb-VLO+O#?F@|20K65^8x}b=0D6=Ui@hgnrMLxGXT(_4 zu6Jb;MoLMLYYRF9r1%E)GbIhMK)9pI`p*)Tt4R+NQ+ zm%uzmxnT~YyH1}AR7$DAz(c_O=mZGpkoz9kC?aPa$!P!%05|)pi@;4Ha^g~rF?h@- zIC!jdI5<6nPLPCcsty6&84re!`P-NwFxg_v?~;1KPfDpb(Yb=wh-R(8Dy7sC7czlw zEp=1MzG6&&Z4NxqW#LAMZYsD z1cn>mdqw1;r5ja4k68GgL+2Q7^UBL!ou5Vk52q3AVc%!rTh^##Ek!W#_=|&XxPqW} z=ULj#x3pX4K>jv^XJ1$k7))?oa3-Pg{WQu_XD(7epZb_|s{{GwnL(X56$AO54y%}T znUmiDXVJSTKQ%d!vDZ@fxTS7(%zvw;A;x!6{EICkUQSXTFMAxs;1dL6`z(6kD-DA4 z(1W1aZk7Z2e;Yjit@Sk6;CZszX>gwd*^N{=4em0&>w()tx>hU_kpl2af?wx5n1b&h zcxYR;t2GVKYhhwr9#=&o@+taAEiNqBHPW-)4)NCj8ZuWbUBn~ z+YCMGZ)0wUZvg(Gyk{H%MvA-b@F4VdcsiHo`Mu~(*JcqhSA5x3ghB>I4#d0kK4@62y(cC>7)BEEH?rRD_ zYsDTQB1eF)jOqcvtBL%O4=o@#!+j}6Mi>b80MCN=5R9LGV0zx(jp6;UgON$r_7RM> zSeR@F8uL~%0`qmiP7Cghq5niMYX>e`n5EvMXJ`Q16`leScM2;pTxww`Kxf{{TCjG9 zV0}CCzY0v7(f9C7rWb%2c`NP1`5^WS61_lEVgq>0JJ2^mOy;Mr1S5FLpRwRON5s8= z*aq-~^n`S;3piZ6Cs4jS#Pb5ojm4u1VT(fq3Hn;=pc*RPi2eFbsk|1OSCxWo8QxOa z0Zgrd^3y`5x>8wQ6~6=k7DWi}2R;GDW}tW*`r$;>RoU2{Piz1WOi#$bSSm%}GxWhm zeF~);(7U?zL|>LZ7dPge7{CM9fxd?q2C7t!dc9&U`m8Q=CF#ZXc(3hh@83%Ndx4f3qxftGHc{DIuF6JW zyc6s$nD`7a2_3Jt3zlh10X?ovbcs2uXLBv+3rTa(Pix|KYm;g5MHKQ zJe(YgPJm7UMx!52-hzHOSvn6C^ux)6=s|5auod`0{0Yn~@MUd)Na5bISrPd!Hw<|i f;7Xut{`L6(SmooGdQ@ Date: Sat, 19 Jul 2025 18:42:39 -0500 Subject: [PATCH 16/44] Remove context length option --- _locales/en-US/messages.json | 3 +-- background.js | 23 +++++++++++++---------- options/options.html | 6 ------ options/options.js | 6 +----- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index fb439b9..65181c8 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -19,8 +19,7 @@ "options.stripUrlParams": { "message": "Remove URL tracking parameters" }, "options.altTextImages": { "message": "Replace images with alt text" }, "options.collapseWhitespace": { "message": "Collapse long whitespace" }, - "options.tokenReduction": { "message": "Aggressive token reduction" }, - "options.contextLength": { "message": "Context length" } + "options.tokenReduction": { "message": "Aggressive token reduction" } ,"action.read": { "message": "read" } ,"action.flag": { "message": "flag" } ,"action.copy": { "message": "copy" } diff --git a/background.js b/background.js index c44bfed..b5667e6 100644 --- a/background.js +++ b/background.js @@ -27,7 +27,7 @@ let stripUrlParams = false; let altTextImages = false; let collapseWhitespace = false; let tokenReduction = false; -let contextLength = 16384; +let maxTokens = 4096; let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; @@ -262,8 +262,8 @@ async function processMessage(id) { try { const full = await messenger.messages.getFull(id); let text = buildEmailText(full); - if (tokenReduction && contextLength > 0) { - const limit = Math.floor(contextLength * 0.9); + if (tokenReduction && maxTokens > 0) { + const limit = Math.floor(maxTokens * 0.9); if (text.length > limit) { text = text.slice(0, limit); } @@ -425,7 +425,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "contextLength", "aiRules", "theme", "errorPending"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -436,7 +436,9 @@ async function clearCacheForMessages(idsInput) { altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; tokenReduction = store.tokenReduction === true; - contextLength = parseInt(store.contextLength) || contextLength; + if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') { + maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; + } errorPending = store.errorPending === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -459,7 +461,12 @@ async function clearCacheForMessages(idsInput) { if (changes.templateName) config.templateName = changes.templateName.newValue; if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; - if (changes.aiParams) config.aiParams = changes.aiParams.newValue; + if (changes.aiParams) { + config.aiParams = changes.aiParams.newValue; + if (changes.aiParams.newValue && typeof changes.aiParams.newValue.max_tokens !== 'undefined') { + maxTokens = parseInt(changes.aiParams.newValue.max_tokens) || maxTokens; + } + } if (changes.debugLogging) { config.debugLogging = changes.debugLogging.newValue === true; logger.setDebug(config.debugLogging); @@ -487,10 +494,6 @@ async function clearCacheForMessages(idsInput) { tokenReduction = changes.tokenReduction.newValue === true; logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction); } - if (changes.contextLength) { - contextLength = parseInt(changes.contextLength.newValue) || contextLength; - logger.aiLog("contextLength updated from storage change", { debug: true }, contextLength); - } if (changes.errorPending) { errorPending = changes.errorPending.newValue === true; updateActionIcon(); diff --git a/options/options.html b/options/options.html index 108d7f9..69158b1 100644 --- a/options/options.html +++ b/options/options.html @@ -149,12 +149,6 @@ Aggressive token reduction
-
- -
- -
-
diff --git a/options/options.js b/options/options.js index f5af1b6..ee2914a 100644 --- a/options/options.js +++ b/options/options.js @@ -17,7 +17,6 @@ document.addEventListener('DOMContentLoaded', async () => { 'altTextImages', 'collapseWhitespace', 'tokenReduction', - 'contextLength', 'aiRules', 'aiCache', 'theme' @@ -120,8 +119,6 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReductionToggle = document.getElementById('token-reduction'); tokenReductionToggle.checked = defaults.tokenReduction === true; - const contextLengthInput = document.getElementById('context-length'); - contextLengthInput.value = defaults.contextLength || 16384; const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { @@ -800,9 +797,8 @@ document.addEventListener('DOMContentLoaded', async () => { const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; const tokenReduction = tokenReductionToggle.checked; - const contextLength = parseInt(contextLengthInput.value) || 0; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, contextLength, aiRules: rules, theme }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme }); await applyTheme(theme); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); From db7ce49a5b47ea495681510396d7cbb26ed26343 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:03:37 -0500 Subject: [PATCH 17/44] Add debug tab with payload viewer --- README.md | 1 + modules/AiClassifier.js | 5 +++++ options/options.html | 14 ++++++++++++++ options/options.js | 21 +++++++++++++++++++-- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3853127..43a515b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ message meets a specified criterion. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. +- **Debug tab** – view the last request payload sent to the AI service. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index b35cb2c..8313654 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -308,6 +308,11 @@ async function classifyText(text, criterion, cacheKey = null) { } const payload = buildPayload(text, criterion); + try { + await storage.local.set({ lastPayload: JSON.parse(payload) }); + } catch (e) { + aiLog('failed to save last payload', { level: 'warn' }, e); + } aiLog(`[AiClassifier] Sending classification request to ${gEndpoint}`, {debug: true}); aiLog(`[AiClassifier] Classification request payload:`, { debug: true }, payload); diff --git a/options/options.html b/options/options.html index 69158b1..d14ce99 100644 --- a/options/options.html +++ b/options/options.html @@ -47,6 +47,7 @@
  • Settings
  • Rules
  • Maintenance
  • +
    @@ -215,6 +216,11 @@
    +
    + +
    @@ -273,6 +279,14 @@

    + + diff --git a/options/options.js b/options/options.js index ee2914a..ebd7497 100644 --- a/options/options.js +++ b/options/options.js @@ -19,7 +19,9 @@ document.addEventListener('DOMContentLoaded', async () => { 'tokenReduction', 'aiRules', 'aiCache', - 'theme' + 'theme', + 'showDebugTab', + 'lastPayload' ]); const tabButtons = document.querySelectorAll('#main-tabs li'); const tabs = document.querySelectorAll('.tab-content'); @@ -64,6 +66,10 @@ document.addEventListener('DOMContentLoaded', async () => { } await applyTheme(themeSelect.value); + const payloadDisplay = document.getElementById('payload-display'); + if (defaults.lastPayload) { + payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2); + } themeSelect.addEventListener('change', async () => { markDirty(); await applyTheme(themeSelect.value); @@ -119,6 +125,16 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReductionToggle = document.getElementById('token-reduction'); tokenReductionToggle.checked = defaults.tokenReduction === true; + const debugTabToggle = document.getElementById('show-debug-tab'); + const debugTabBtn = document.getElementById('debug-tab-button'); + function updateDebugTab() { + const visible = debugTabToggle.checked; + debugTabBtn.classList.toggle('is-hidden', !visible); + } + debugTabToggle.checked = defaults.showDebugTab === true; + debugTabToggle.addEventListener('change', () => { updateDebugTab(); markDirty(); }); + updateDebugTab(); + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { @@ -797,8 +813,9 @@ document.addEventListener('DOMContentLoaded', async () => { const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; const tokenReduction = tokenReductionToggle.checked; + const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); await applyTheme(theme); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); From d2de1818ca11956dfa73bb21cddc0feeb4e8c349 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:24:47 -0500 Subject: [PATCH 18/44] Adding diff library --- README.md | 2 + ai-filter.sln | 1 + resources/js/diff_match_patch_uncompressed.js | 2236 +++++++++++++++++ 3 files changed, 2239 insertions(+) create mode 100644 resources/js/diff_match_patch_uncompressed.js diff --git a/README.md b/README.md index 43a515b..64fbf26 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,8 @@ uses the following third party libraries: - MIT License - [turndown v7.2.0](https://github.com/mixmark-io/turndown/tree/v7.2.0) - MIT License +- [diff](https://github.com/google/diff-match-patch/blob/62f2e689f498f9c92dbc588c58750addec9b1654/javascript/diff_match_patch_uncompressed.js) + - Apache-2.0 license ## License diff --git a/ai-filter.sln b/ai-filter.sln index 57705eb..ea71f1b 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -108,6 +108,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-175 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "js", "js", "{21D2A42C-3F85-465C-9141-C106AFD92B68}" ProjectSection(SolutionItems) = preProject + resources\js\diff_match_patch_uncompressed.js = resources\js\diff_match_patch_uncompressed.js resources\js\turndown.js = resources\js\turndown.js EndProjectSection EndProject diff --git a/resources/js/diff_match_patch_uncompressed.js b/resources/js/diff_match_patch_uncompressed.js new file mode 100644 index 0000000..88a702c --- /dev/null +++ b/resources/js/diff_match_patch_uncompressed.js @@ -0,0 +1,2236 @@ +/** + * Diff Match and Patch + * Copyright 2018 The diff-match-patch Authors. + * https://github.com/google/diff-match-patch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Computes the difference between two texts to create a patch. + * Applies the patch onto another text, allowing for errors. + * @author fraser@google.com (Neil Fraser) + */ + +/** + * Class containing the diff, match and patch methods. + * @constructor + */ +var diff_match_patch = function() { + + // Defaults. + // Redefine these in your program to override the defaults. + + // Number of seconds to map a diff before giving up (0 for infinity). + this.Diff_Timeout = 1.0; + // Cost of an empty edit operation in terms of edit characters. + this.Diff_EditCost = 4; + // At what point is no match declared (0.0 = perfection, 1.0 = very loose). + this.Match_Threshold = 0.5; + // How far to search for a match (0 = exact location, 1000+ = broad match). + // A match this many characters away from the expected location will add + // 1.0 to the score (0.0 is a perfect match). + this.Match_Distance = 1000; + // When deleting a large block of text (over ~64 characters), how close do + // the contents have to be to match the expected contents. (0.0 = perfection, + // 1.0 = very loose). Note that Match_Threshold controls how closely the + // end points of a delete need to match. + this.Patch_DeleteThreshold = 0.5; + // Chunk size for context length. + this.Patch_Margin = 4; + + // The number of bits in an int. + this.Match_MaxBits = 32; +}; + + +// DIFF FUNCTIONS + + +/** + * The data structure representing a diff is an array of tuples: + * [[DIFF_DELETE, 'Hello'], [DIFF_INSERT, 'Goodbye'], [DIFF_EQUAL, ' world.']] + * which means: delete 'Hello', add 'Goodbye' and keep ' world.' + */ +var DIFF_DELETE = -1; +var DIFF_INSERT = 1; +var DIFF_EQUAL = 0; + +/** + * Class representing one diff tuple. + * Attempts to look like a two-element array (which is what this used to be). + * @param {number} op Operation, one of: DIFF_DELETE, DIFF_INSERT, DIFF_EQUAL. + * @param {string} text Text to be deleted, inserted, or retained. + * @constructor + */ +diff_match_patch.Diff = function(op, text) { + this[0] = op; + this[1] = text; +}; + +diff_match_patch.Diff.prototype.length = 2; + +/** + * Emulate the output of a two-element array. + * @return {string} Diff operation as a string. + */ +diff_match_patch.Diff.prototype.toString = function() { + return this[0] + ',' + this[1]; +}; + + +/** + * Find the differences between two texts. Simplifies the problem by stripping + * any common prefix or suffix off the texts before diffing. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {boolean=} opt_checklines Optional speedup flag. If present and false, + * then don't run a line-level diff first to identify the changed areas. + * Defaults to true, which does a faster, slightly less optimal diff. + * @param {number=} opt_deadline Optional time when the diff should be complete + * by. Used internally for recursive calls. Users should set DiffTimeout + * instead. + * @return {!Array.} Array of diff tuples. + */ +diff_match_patch.prototype.diff_main = function(text1, text2, opt_checklines, + opt_deadline) { + // Set a deadline by which time the diff must be complete. + if (typeof opt_deadline == 'undefined') { + if (this.Diff_Timeout <= 0) { + opt_deadline = Number.MAX_VALUE; + } else { + opt_deadline = (new Date).getTime() + this.Diff_Timeout * 1000; + } + } + var deadline = opt_deadline; + + // Check for null inputs. + if (text1 == null || text2 == null) { + throw new Error('Null input. (diff_main)'); + } + + // Check for equality (speedup). + if (text1 == text2) { + if (text1) { + return [new diff_match_patch.Diff(DIFF_EQUAL, text1)]; + } + return []; + } + + if (typeof opt_checklines == 'undefined') { + opt_checklines = true; + } + var checklines = opt_checklines; + + // Trim off common prefix (speedup). + var commonlength = this.diff_commonPrefix(text1, text2); + var commonprefix = text1.substring(0, commonlength); + text1 = text1.substring(commonlength); + text2 = text2.substring(commonlength); + + // Trim off common suffix (speedup). + commonlength = this.diff_commonSuffix(text1, text2); + var commonsuffix = text1.substring(text1.length - commonlength); + text1 = text1.substring(0, text1.length - commonlength); + text2 = text2.substring(0, text2.length - commonlength); + + // Compute the diff on the middle block. + var diffs = this.diff_compute_(text1, text2, checklines, deadline); + + // Restore the prefix and suffix. + if (commonprefix) { + diffs.unshift(new diff_match_patch.Diff(DIFF_EQUAL, commonprefix)); + } + if (commonsuffix) { + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, commonsuffix)); + } + this.diff_cleanupMerge(diffs); + return diffs; +}; + + +/** + * Find the differences between two texts. Assumes that the texts do not + * have any common prefix or suffix. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {boolean} checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster, slightly less optimal diff. + * @param {number} deadline Time when the diff should be complete by. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_compute_ = function(text1, text2, checklines, + deadline) { + var diffs; + + if (!text1) { + // Just add some text (speedup). + return [new diff_match_patch.Diff(DIFF_INSERT, text2)]; + } + + if (!text2) { + // Just delete some text (speedup). + return [new diff_match_patch.Diff(DIFF_DELETE, text1)]; + } + + var longtext = text1.length > text2.length ? text1 : text2; + var shorttext = text1.length > text2.length ? text2 : text1; + var i = longtext.indexOf(shorttext); + if (i != -1) { + // Shorter text is inside the longer text (speedup). + diffs = [new diff_match_patch.Diff(DIFF_INSERT, longtext.substring(0, i)), + new diff_match_patch.Diff(DIFF_EQUAL, shorttext), + new diff_match_patch.Diff(DIFF_INSERT, + longtext.substring(i + shorttext.length))]; + // Swap insertions for deletions if diff is reversed. + if (text1.length > text2.length) { + diffs[0][0] = diffs[2][0] = DIFF_DELETE; + } + return diffs; + } + + if (shorttext.length == 1) { + // Single character string. + // After the previous speedup, the character can't be an equality. + return [new diff_match_patch.Diff(DIFF_DELETE, text1), + new diff_match_patch.Diff(DIFF_INSERT, text2)]; + } + + // Check to see if the problem can be split in two. + var hm = this.diff_halfMatch_(text1, text2); + if (hm) { + // A half-match was found, sort out the return data. + var text1_a = hm[0]; + var text1_b = hm[1]; + var text2_a = hm[2]; + var text2_b = hm[3]; + var mid_common = hm[4]; + // Send both pairs off for separate processing. + var diffs_a = this.diff_main(text1_a, text2_a, checklines, deadline); + var diffs_b = this.diff_main(text1_b, text2_b, checklines, deadline); + // Merge the results. + return diffs_a.concat([new diff_match_patch.Diff(DIFF_EQUAL, mid_common)], + diffs_b); + } + + if (checklines && text1.length > 100 && text2.length > 100) { + return this.diff_lineMode_(text1, text2, deadline); + } + + return this.diff_bisect_(text1, text2, deadline); +}; + + +/** + * Do a quick line-level diff on both strings, then rediff the parts for + * greater accuracy. + * This speedup can produce non-minimal diffs. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} deadline Time when the diff should be complete by. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_lineMode_ = function(text1, text2, deadline) { + // Scan the text on a line-by-line basis first. + var a = this.diff_linesToChars_(text1, text2); + text1 = a.chars1; + text2 = a.chars2; + var linearray = a.lineArray; + + var diffs = this.diff_main(text1, text2, false, deadline); + + // Convert the diff back to original text. + this.diff_charsToLines_(diffs, linearray); + // Eliminate freak matches (e.g. blank lines) + this.diff_cleanupSemantic(diffs); + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, '')); + var pointer = 0; + var count_delete = 0; + var count_insert = 0; + var text_delete = ''; + var text_insert = ''; + while (pointer < diffs.length) { + switch (diffs[pointer][0]) { + case DIFF_INSERT: + count_insert++; + text_insert += diffs[pointer][1]; + break; + case DIFF_DELETE: + count_delete++; + text_delete += diffs[pointer][1]; + break; + case DIFF_EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete >= 1 && count_insert >= 1) { + // Delete the offending records and add the merged ones. + diffs.splice(pointer - count_delete - count_insert, + count_delete + count_insert); + pointer = pointer - count_delete - count_insert; + var subDiff = + this.diff_main(text_delete, text_insert, false, deadline); + for (var j = subDiff.length - 1; j >= 0; j--) { + diffs.splice(pointer, 0, subDiff[j]); + } + pointer = pointer + subDiff.length; + } + count_insert = 0; + count_delete = 0; + text_delete = ''; + text_insert = ''; + break; + } + pointer++; + } + diffs.pop(); // Remove the dummy entry at the end. + + return diffs; +}; + + +/** + * Find the 'middle snake' of a diff, split the problem in two + * and return the recursively constructed diff. + * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} deadline Time at which to bail if not yet complete. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_bisect_ = function(text1, text2, deadline) { + // Cache the text lengths to prevent multiple calls. + var text1_length = text1.length; + var text2_length = text2.length; + var max_d = Math.ceil((text1_length + text2_length) / 2); + var v_offset = max_d; + var v_length = 2 * max_d; + var v1 = new Array(v_length); + var v2 = new Array(v_length); + // Setting all elements to -1 is faster in Chrome & Firefox than mixing + // integers and undefined. + for (var x = 0; x < v_length; x++) { + v1[x] = -1; + v2[x] = -1; + } + v1[v_offset + 1] = 0; + v2[v_offset + 1] = 0; + var delta = text1_length - text2_length; + // If the total number of characters is odd, then the front path will collide + // with the reverse path. + var front = (delta % 2 != 0); + // Offsets for start and end of k loop. + // Prevents mapping of space beyond the grid. + var k1start = 0; + var k1end = 0; + var k2start = 0; + var k2end = 0; + for (var d = 0; d < max_d; d++) { + // Bail out if deadline is reached. + if ((new Date()).getTime() > deadline) { + break; + } + + // Walk the front path one step. + for (var k1 = -d + k1start; k1 <= d - k1end; k1 += 2) { + var k1_offset = v_offset + k1; + var x1; + if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) { + x1 = v1[k1_offset + 1]; + } else { + x1 = v1[k1_offset - 1] + 1; + } + var y1 = x1 - k1; + while (x1 < text1_length && y1 < text2_length && + text1.charAt(x1) == text2.charAt(y1)) { + x1++; + y1++; + } + v1[k1_offset] = x1; + if (x1 > text1_length) { + // Ran off the right of the graph. + k1end += 2; + } else if (y1 > text2_length) { + // Ran off the bottom of the graph. + k1start += 2; + } else if (front) { + var k2_offset = v_offset + delta - k1; + if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) { + // Mirror x2 onto top-left coordinate system. + var x2 = text1_length - v2[k2_offset]; + if (x1 >= x2) { + // Overlap detected. + return this.diff_bisectSplit_(text1, text2, x1, y1, deadline); + } + } + } + } + + // Walk the reverse path one step. + for (var k2 = -d + k2start; k2 <= d - k2end; k2 += 2) { + var k2_offset = v_offset + k2; + var x2; + if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) { + x2 = v2[k2_offset + 1]; + } else { + x2 = v2[k2_offset - 1] + 1; + } + var y2 = x2 - k2; + while (x2 < text1_length && y2 < text2_length && + text1.charAt(text1_length - x2 - 1) == + text2.charAt(text2_length - y2 - 1)) { + x2++; + y2++; + } + v2[k2_offset] = x2; + if (x2 > text1_length) { + // Ran off the left of the graph. + k2end += 2; + } else if (y2 > text2_length) { + // Ran off the top of the graph. + k2start += 2; + } else if (!front) { + var k1_offset = v_offset + delta - k2; + if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) { + var x1 = v1[k1_offset]; + var y1 = v_offset + x1 - k1_offset; + // Mirror x2 onto top-left coordinate system. + x2 = text1_length - x2; + if (x1 >= x2) { + // Overlap detected. + return this.diff_bisectSplit_(text1, text2, x1, y1, deadline); + } + } + } + } + } + // Diff took too long and hit the deadline or + // number of diffs equals number of characters, no commonality at all. + return [new diff_match_patch.Diff(DIFF_DELETE, text1), + new diff_match_patch.Diff(DIFF_INSERT, text2)]; +}; + + +/** + * Given the location of the 'middle snake', split the diff in two parts + * and recurse. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} x Index of split point in text1. + * @param {number} y Index of split point in text2. + * @param {number} deadline Time at which to bail if not yet complete. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_bisectSplit_ = function(text1, text2, x, y, + deadline) { + var text1a = text1.substring(0, x); + var text2a = text2.substring(0, y); + var text1b = text1.substring(x); + var text2b = text2.substring(y); + + // Compute both diffs serially. + var diffs = this.diff_main(text1a, text2a, false, deadline); + var diffsb = this.diff_main(text1b, text2b, false, deadline); + + return diffs.concat(diffsb); +}; + + +/** + * Split two texts into an array of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {{chars1: string, chars2: string, lineArray: !Array.}} + * An object containing the encoded text1, the encoded text2 and + * the array of unique strings. + * The zeroth element of the array of unique strings is intentionally blank. + * @private + */ +diff_match_patch.prototype.diff_linesToChars_ = function(text1, text2) { + var lineArray = []; // e.g. lineArray[4] == 'Hello\n' + var lineHash = {}; // e.g. lineHash['Hello\n'] == 4 + + // '\x00' is a valid character, but various debuggers don't like it. + // So we'll insert a junk entry to avoid generating a null character. + lineArray[0] = ''; + + /** + * Split a text into an array of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * Modifies linearray and linehash through being a closure. + * @param {string} text String to encode. + * @return {string} Encoded string. + * @private + */ + function diff_linesToCharsMunge_(text) { + var chars = ''; + // Walk the text, pulling out a substring for each line. + // text.split('\n') would would temporarily double our memory footprint. + // Modifying text would create many large strings to garbage collect. + var lineStart = 0; + var lineEnd = -1; + // Keeping our own length variable is faster than looking it up. + var lineArrayLength = lineArray.length; + while (lineEnd < text.length - 1) { + lineEnd = text.indexOf('\n', lineStart); + if (lineEnd == -1) { + lineEnd = text.length - 1; + } + var line = text.substring(lineStart, lineEnd + 1); + + if (lineHash.hasOwnProperty ? lineHash.hasOwnProperty(line) : + (lineHash[line] !== undefined)) { + chars += String.fromCharCode(lineHash[line]); + } else { + if (lineArrayLength == maxLines) { + // Bail out at 65535 because + // String.fromCharCode(65536) == String.fromCharCode(0) + line = text.substring(lineStart); + lineEnd = text.length; + } + chars += String.fromCharCode(lineArrayLength); + lineHash[line] = lineArrayLength; + lineArray[lineArrayLength++] = line; + } + lineStart = lineEnd + 1; + } + return chars; + } + // Allocate 2/3rds of the space for text1, the rest for text2. + var maxLines = 40000; + var chars1 = diff_linesToCharsMunge_(text1); + maxLines = 65535; + var chars2 = diff_linesToCharsMunge_(text2); + return {chars1: chars1, chars2: chars2, lineArray: lineArray}; +}; + + +/** + * Rehydrate the text in a diff from a string of line hashes to real lines of + * text. + * @param {!Array.} diffs Array of diff tuples. + * @param {!Array.} lineArray Array of unique strings. + * @private + */ +diff_match_patch.prototype.diff_charsToLines_ = function(diffs, lineArray) { + for (var i = 0; i < diffs.length; i++) { + var chars = diffs[i][1]; + var text = []; + for (var j = 0; j < chars.length; j++) { + text[j] = lineArray[chars.charCodeAt(j)]; + } + diffs[i][1] = text.join(''); + } +}; + + +/** + * Determine the common prefix of two strings. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the start of each + * string. + */ +diff_match_patch.prototype.diff_commonPrefix = function(text1, text2) { + // Quick check for common null cases. + if (!text1 || !text2 || text1.charAt(0) != text2.charAt(0)) { + return 0; + } + // Binary search. + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + var pointermin = 0; + var pointermax = Math.min(text1.length, text2.length); + var pointermid = pointermax; + var pointerstart = 0; + while (pointermin < pointermid) { + if (text1.substring(pointerstart, pointermid) == + text2.substring(pointerstart, pointermid)) { + pointermin = pointermid; + pointerstart = pointermin; + } else { + pointermax = pointermid; + } + pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin); + } + return pointermid; +}; + + +/** + * Determine the common suffix of two strings. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the end of each string. + */ +diff_match_patch.prototype.diff_commonSuffix = function(text1, text2) { + // Quick check for common null cases. + if (!text1 || !text2 || + text1.charAt(text1.length - 1) != text2.charAt(text2.length - 1)) { + return 0; + } + // Binary search. + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + var pointermin = 0; + var pointermax = Math.min(text1.length, text2.length); + var pointermid = pointermax; + var pointerend = 0; + while (pointermin < pointermid) { + if (text1.substring(text1.length - pointermid, text1.length - pointerend) == + text2.substring(text2.length - pointermid, text2.length - pointerend)) { + pointermin = pointermid; + pointerend = pointermin; + } else { + pointermax = pointermid; + } + pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin); + } + return pointermid; +}; + + +/** + * Determine if the suffix of one string is the prefix of another. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the end of the first + * string and the start of the second string. + * @private + */ +diff_match_patch.prototype.diff_commonOverlap_ = function(text1, text2) { + // Cache the text lengths to prevent multiple calls. + var text1_length = text1.length; + var text2_length = text2.length; + // Eliminate the null case. + if (text1_length == 0 || text2_length == 0) { + return 0; + } + // Truncate the longer string. + if (text1_length > text2_length) { + text1 = text1.substring(text1_length - text2_length); + } else if (text1_length < text2_length) { + text2 = text2.substring(0, text1_length); + } + var text_length = Math.min(text1_length, text2_length); + // Quick check for the worst case. + if (text1 == text2) { + return text_length; + } + + // Start by looking for a single character match + // and increase length until no match is found. + // Performance analysis: https://neil.fraser.name/news/2010/11/04/ + var best = 0; + var length = 1; + while (true) { + var pattern = text1.substring(text_length - length); + var found = text2.indexOf(pattern); + if (found == -1) { + return best; + } + length += found; + if (found == 0 || text1.substring(text_length - length) == + text2.substring(0, length)) { + best = length; + length++; + } + } +}; + + +/** + * Do the two texts share a substring which is at least half the length of the + * longer text? + * This speedup can produce non-minimal diffs. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {Array.} Five element Array, containing the prefix of + * text1, the suffix of text1, the prefix of text2, the suffix of + * text2 and the common middle. Or null if there was no match. + * @private + */ +diff_match_patch.prototype.diff_halfMatch_ = function(text1, text2) { + if (this.Diff_Timeout <= 0) { + // Don't risk returning a non-optimal diff if we have unlimited time. + return null; + } + var longtext = text1.length > text2.length ? text1 : text2; + var shorttext = text1.length > text2.length ? text2 : text1; + if (longtext.length < 4 || shorttext.length * 2 < longtext.length) { + return null; // Pointless. + } + var dmp = this; // 'this' becomes 'window' in a closure. + + /** + * Does a substring of shorttext exist within longtext such that the substring + * is at least half the length of longtext? + * Closure, but does not reference any external variables. + * @param {string} longtext Longer string. + * @param {string} shorttext Shorter string. + * @param {number} i Start index of quarter length substring within longtext. + * @return {Array.} Five element Array, containing the prefix of + * longtext, the suffix of longtext, the prefix of shorttext, the suffix + * of shorttext and the common middle. Or null if there was no match. + * @private + */ + function diff_halfMatchI_(longtext, shorttext, i) { + // Start with a 1/4 length substring at position i as a seed. + var seed = longtext.substring(i, i + Math.floor(longtext.length / 4)); + var j = -1; + var best_common = ''; + var best_longtext_a, best_longtext_b, best_shorttext_a, best_shorttext_b; + while ((j = shorttext.indexOf(seed, j + 1)) != -1) { + var prefixLength = dmp.diff_commonPrefix(longtext.substring(i), + shorttext.substring(j)); + var suffixLength = dmp.diff_commonSuffix(longtext.substring(0, i), + shorttext.substring(0, j)); + if (best_common.length < suffixLength + prefixLength) { + best_common = shorttext.substring(j - suffixLength, j) + + shorttext.substring(j, j + prefixLength); + best_longtext_a = longtext.substring(0, i - suffixLength); + best_longtext_b = longtext.substring(i + prefixLength); + best_shorttext_a = shorttext.substring(0, j - suffixLength); + best_shorttext_b = shorttext.substring(j + prefixLength); + } + } + if (best_common.length * 2 >= longtext.length) { + return [best_longtext_a, best_longtext_b, + best_shorttext_a, best_shorttext_b, best_common]; + } else { + return null; + } + } + + // First check if the second quarter is the seed for a half-match. + var hm1 = diff_halfMatchI_(longtext, shorttext, + Math.ceil(longtext.length / 4)); + // Check again based on the third quarter. + var hm2 = diff_halfMatchI_(longtext, shorttext, + Math.ceil(longtext.length / 2)); + var hm; + if (!hm1 && !hm2) { + return null; + } else if (!hm2) { + hm = hm1; + } else if (!hm1) { + hm = hm2; + } else { + // Both matched. Select the longest. + hm = hm1[4].length > hm2[4].length ? hm1 : hm2; + } + + // A half-match was found, sort out the return data. + var text1_a, text1_b, text2_a, text2_b; + if (text1.length > text2.length) { + text1_a = hm[0]; + text1_b = hm[1]; + text2_a = hm[2]; + text2_b = hm[3]; + } else { + text2_a = hm[0]; + text2_b = hm[1]; + text1_a = hm[2]; + text1_b = hm[3]; + } + var mid_common = hm[4]; + return [text1_a, text1_b, text2_a, text2_b, mid_common]; +}; + + +/** + * Reduce the number of edits by eliminating semantically trivial equalities. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupSemantic = function(diffs) { + var changes = false; + var equalities = []; // Stack of indices where equalities are found. + var equalitiesLength = 0; // Keeping our own length var is faster in JS. + /** @type {?string} */ + var lastEquality = null; + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer = 0; // Index of current position. + // Number of characters that changed prior to the equality. + var length_insertions1 = 0; + var length_deletions1 = 0; + // Number of characters that changed after the equality. + var length_insertions2 = 0; + var length_deletions2 = 0; + while (pointer < diffs.length) { + if (diffs[pointer][0] == DIFF_EQUAL) { // Equality found. + equalities[equalitiesLength++] = pointer; + length_insertions1 = length_insertions2; + length_deletions1 = length_deletions2; + length_insertions2 = 0; + length_deletions2 = 0; + lastEquality = diffs[pointer][1]; + } else { // An insertion or deletion. + if (diffs[pointer][0] == DIFF_INSERT) { + length_insertions2 += diffs[pointer][1].length; + } else { + length_deletions2 += diffs[pointer][1].length; + } + // Eliminate an equality that is smaller or equal to the edits on both + // sides of it. + if (lastEquality && (lastEquality.length <= + Math.max(length_insertions1, length_deletions1)) && + (lastEquality.length <= Math.max(length_insertions2, + length_deletions2))) { + // Duplicate record. + diffs.splice(equalities[equalitiesLength - 1], 0, + new diff_match_patch.Diff(DIFF_DELETE, lastEquality)); + // Change second copy to insert. + diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT; + // Throw away the equality we just deleted. + equalitiesLength--; + // Throw away the previous equality (it needs to be reevaluated). + equalitiesLength--; + pointer = equalitiesLength > 0 ? equalities[equalitiesLength - 1] : -1; + length_insertions1 = 0; // Reset the counters. + length_deletions1 = 0; + length_insertions2 = 0; + length_deletions2 = 0; + lastEquality = null; + changes = true; + } + } + pointer++; + } + + // Normalize the diff. + if (changes) { + this.diff_cleanupMerge(diffs); + } + this.diff_cleanupSemanticLossless(diffs); + + // Find any overlaps between deletions and insertions. + // e.g: abcxxxxxxdef + // -> abcxxxdef + // e.g: xxxabcdefxxx + // -> defxxxabc + // Only extract an overlap if it is as big as the edit ahead or behind it. + pointer = 1; + while (pointer < diffs.length) { + if (diffs[pointer - 1][0] == DIFF_DELETE && + diffs[pointer][0] == DIFF_INSERT) { + var deletion = diffs[pointer - 1][1]; + var insertion = diffs[pointer][1]; + var overlap_length1 = this.diff_commonOverlap_(deletion, insertion); + var overlap_length2 = this.diff_commonOverlap_(insertion, deletion); + if (overlap_length1 >= overlap_length2) { + if (overlap_length1 >= deletion.length / 2 || + overlap_length1 >= insertion.length / 2) { + // Overlap found. Insert an equality and trim the surrounding edits. + diffs.splice(pointer, 0, new diff_match_patch.Diff(DIFF_EQUAL, + insertion.substring(0, overlap_length1))); + diffs[pointer - 1][1] = + deletion.substring(0, deletion.length - overlap_length1); + diffs[pointer + 1][1] = insertion.substring(overlap_length1); + pointer++; + } + } else { + if (overlap_length2 >= deletion.length / 2 || + overlap_length2 >= insertion.length / 2) { + // Reverse overlap found. + // Insert an equality and swap and trim the surrounding edits. + diffs.splice(pointer, 0, new diff_match_patch.Diff(DIFF_EQUAL, + deletion.substring(0, overlap_length2))); + diffs[pointer - 1][0] = DIFF_INSERT; + diffs[pointer - 1][1] = + insertion.substring(0, insertion.length - overlap_length2); + diffs[pointer + 1][0] = DIFF_DELETE; + diffs[pointer + 1][1] = + deletion.substring(overlap_length2); + pointer++; + } + } + pointer++; + } + pointer++; + } +}; + + +/** + * Look for single edits surrounded on both sides by equalities + * which can be shifted sideways to align the edit to a word boundary. + * e.g: The cat came. -> The cat came. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupSemanticLossless = function(diffs) { + /** + * Given two strings, compute a score representing whether the internal + * boundary falls on logical boundaries. + * Scores range from 6 (best) to 0 (worst). + * Closure, but does not reference any external variables. + * @param {string} one First string. + * @param {string} two Second string. + * @return {number} The score. + * @private + */ + function diff_cleanupSemanticScore_(one, two) { + if (!one || !two) { + // Edges are the best. + return 6; + } + + // Each port of this function behaves slightly differently due to + // subtle differences in each language's definition of things like + // 'whitespace'. Since this function's purpose is largely cosmetic, + // the choice has been made to use each language's native features + // rather than force total conformity. + var char1 = one.charAt(one.length - 1); + var char2 = two.charAt(0); + var nonAlphaNumeric1 = char1.match(diff_match_patch.nonAlphaNumericRegex_); + var nonAlphaNumeric2 = char2.match(diff_match_patch.nonAlphaNumericRegex_); + var whitespace1 = nonAlphaNumeric1 && + char1.match(diff_match_patch.whitespaceRegex_); + var whitespace2 = nonAlphaNumeric2 && + char2.match(diff_match_patch.whitespaceRegex_); + var lineBreak1 = whitespace1 && + char1.match(diff_match_patch.linebreakRegex_); + var lineBreak2 = whitespace2 && + char2.match(diff_match_patch.linebreakRegex_); + var blankLine1 = lineBreak1 && + one.match(diff_match_patch.blanklineEndRegex_); + var blankLine2 = lineBreak2 && + two.match(diff_match_patch.blanklineStartRegex_); + + if (blankLine1 || blankLine2) { + // Five points for blank lines. + return 5; + } else if (lineBreak1 || lineBreak2) { + // Four points for line breaks. + return 4; + } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) { + // Three points for end of sentences. + return 3; + } else if (whitespace1 || whitespace2) { + // Two points for whitespace. + return 2; + } else if (nonAlphaNumeric1 || nonAlphaNumeric2) { + // One point for non-alphanumeric. + return 1; + } + return 0; + } + + var pointer = 1; + // Intentionally ignore the first and last element (don't need checking). + while (pointer < diffs.length - 1) { + if (diffs[pointer - 1][0] == DIFF_EQUAL && + diffs[pointer + 1][0] == DIFF_EQUAL) { + // This is a single edit surrounded by equalities. + var equality1 = diffs[pointer - 1][1]; + var edit = diffs[pointer][1]; + var equality2 = diffs[pointer + 1][1]; + + // First, shift the edit as far left as possible. + var commonOffset = this.diff_commonSuffix(equality1, edit); + if (commonOffset) { + var commonString = edit.substring(edit.length - commonOffset); + equality1 = equality1.substring(0, equality1.length - commonOffset); + edit = commonString + edit.substring(0, edit.length - commonOffset); + equality2 = commonString + equality2; + } + + // Second, step character by character right, looking for the best fit. + var bestEquality1 = equality1; + var bestEdit = edit; + var bestEquality2 = equality2; + var bestScore = diff_cleanupSemanticScore_(equality1, edit) + + diff_cleanupSemanticScore_(edit, equality2); + while (edit.charAt(0) === equality2.charAt(0)) { + equality1 += edit.charAt(0); + edit = edit.substring(1) + equality2.charAt(0); + equality2 = equality2.substring(1); + var score = diff_cleanupSemanticScore_(equality1, edit) + + diff_cleanupSemanticScore_(edit, equality2); + // The >= encourages trailing rather than leading whitespace on edits. + if (score >= bestScore) { + bestScore = score; + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + } + } + + if (diffs[pointer - 1][1] != bestEquality1) { + // We have an improvement, save it back to the diff. + if (bestEquality1) { + diffs[pointer - 1][1] = bestEquality1; + } else { + diffs.splice(pointer - 1, 1); + pointer--; + } + diffs[pointer][1] = bestEdit; + if (bestEquality2) { + diffs[pointer + 1][1] = bestEquality2; + } else { + diffs.splice(pointer + 1, 1); + pointer--; + } + } + } + pointer++; + } +}; + +// Define some regex patterns for matching boundaries. +diff_match_patch.nonAlphaNumericRegex_ = /[^a-zA-Z0-9]/; +diff_match_patch.whitespaceRegex_ = /\s/; +diff_match_patch.linebreakRegex_ = /[\r\n]/; +diff_match_patch.blanklineEndRegex_ = /\n\r?\n$/; +diff_match_patch.blanklineStartRegex_ = /^\r?\n\r?\n/; + +/** + * Reduce the number of edits by eliminating operationally trivial equalities. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupEfficiency = function(diffs) { + var changes = false; + var equalities = []; // Stack of indices where equalities are found. + var equalitiesLength = 0; // Keeping our own length var is faster in JS. + /** @type {?string} */ + var lastEquality = null; + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer = 0; // Index of current position. + // Is there an insertion operation before the last equality. + var pre_ins = false; + // Is there a deletion operation before the last equality. + var pre_del = false; + // Is there an insertion operation after the last equality. + var post_ins = false; + // Is there a deletion operation after the last equality. + var post_del = false; + while (pointer < diffs.length) { + if (diffs[pointer][0] == DIFF_EQUAL) { // Equality found. + if (diffs[pointer][1].length < this.Diff_EditCost && + (post_ins || post_del)) { + // Candidate found. + equalities[equalitiesLength++] = pointer; + pre_ins = post_ins; + pre_del = post_del; + lastEquality = diffs[pointer][1]; + } else { + // Not a candidate, and can never become one. + equalitiesLength = 0; + lastEquality = null; + } + post_ins = post_del = false; + } else { // An insertion or deletion. + if (diffs[pointer][0] == DIFF_DELETE) { + post_del = true; + } else { + post_ins = true; + } + /* + * Five types to be split: + * ABXYCD + * AXCD + * ABXC + * AXCD + * ABXC + */ + if (lastEquality && ((pre_ins && pre_del && post_ins && post_del) || + ((lastEquality.length < this.Diff_EditCost / 2) && + (pre_ins + pre_del + post_ins + post_del) == 3))) { + // Duplicate record. + diffs.splice(equalities[equalitiesLength - 1], 0, + new diff_match_patch.Diff(DIFF_DELETE, lastEquality)); + // Change second copy to insert. + diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT; + equalitiesLength--; // Throw away the equality we just deleted; + lastEquality = null; + if (pre_ins && pre_del) { + // No changes made which could affect previous entry, keep going. + post_ins = post_del = true; + equalitiesLength = 0; + } else { + equalitiesLength--; // Throw away the previous equality. + pointer = equalitiesLength > 0 ? + equalities[equalitiesLength - 1] : -1; + post_ins = post_del = false; + } + changes = true; + } + } + pointer++; + } + + if (changes) { + this.diff_cleanupMerge(diffs); + } +}; + + +/** + * Reorder and merge like edit sections. Merge equalities. + * Any edit section can move as long as it doesn't cross an equality. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupMerge = function(diffs) { + // Add a dummy entry at the end. + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, '')); + var pointer = 0; + var count_delete = 0; + var count_insert = 0; + var text_delete = ''; + var text_insert = ''; + var commonlength; + while (pointer < diffs.length) { + switch (diffs[pointer][0]) { + case DIFF_INSERT: + count_insert++; + text_insert += diffs[pointer][1]; + pointer++; + break; + case DIFF_DELETE: + count_delete++; + text_delete += diffs[pointer][1]; + pointer++; + break; + case DIFF_EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete + count_insert > 1) { + if (count_delete !== 0 && count_insert !== 0) { + // Factor out any common prefixies. + commonlength = this.diff_commonPrefix(text_insert, text_delete); + if (commonlength !== 0) { + if ((pointer - count_delete - count_insert) > 0 && + diffs[pointer - count_delete - count_insert - 1][0] == + DIFF_EQUAL) { + diffs[pointer - count_delete - count_insert - 1][1] += + text_insert.substring(0, commonlength); + } else { + diffs.splice(0, 0, new diff_match_patch.Diff(DIFF_EQUAL, + text_insert.substring(0, commonlength))); + pointer++; + } + text_insert = text_insert.substring(commonlength); + text_delete = text_delete.substring(commonlength); + } + // Factor out any common suffixies. + commonlength = this.diff_commonSuffix(text_insert, text_delete); + if (commonlength !== 0) { + diffs[pointer][1] = text_insert.substring(text_insert.length - + commonlength) + diffs[pointer][1]; + text_insert = text_insert.substring(0, text_insert.length - + commonlength); + text_delete = text_delete.substring(0, text_delete.length - + commonlength); + } + } + // Delete the offending records and add the merged ones. + pointer -= count_delete + count_insert; + diffs.splice(pointer, count_delete + count_insert); + if (text_delete.length) { + diffs.splice(pointer, 0, + new diff_match_patch.Diff(DIFF_DELETE, text_delete)); + pointer++; + } + if (text_insert.length) { + diffs.splice(pointer, 0, + new diff_match_patch.Diff(DIFF_INSERT, text_insert)); + pointer++; + } + pointer++; + } else if (pointer !== 0 && diffs[pointer - 1][0] == DIFF_EQUAL) { + // Merge this equality with the previous one. + diffs[pointer - 1][1] += diffs[pointer][1]; + diffs.splice(pointer, 1); + } else { + pointer++; + } + count_insert = 0; + count_delete = 0; + text_delete = ''; + text_insert = ''; + break; + } + } + if (diffs[diffs.length - 1][1] === '') { + diffs.pop(); // Remove the dummy entry at the end. + } + + // Second pass: look for single edits surrounded on both sides by equalities + // which can be shifted sideways to eliminate an equality. + // e.g: ABAC -> ABAC + var changes = false; + pointer = 1; + // Intentionally ignore the first and last element (don't need checking). + while (pointer < diffs.length - 1) { + if (diffs[pointer - 1][0] == DIFF_EQUAL && + diffs[pointer + 1][0] == DIFF_EQUAL) { + // This is a single edit surrounded by equalities. + if (diffs[pointer][1].substring(diffs[pointer][1].length - + diffs[pointer - 1][1].length) == diffs[pointer - 1][1]) { + // Shift the edit over the previous equality. + diffs[pointer][1] = diffs[pointer - 1][1] + + diffs[pointer][1].substring(0, diffs[pointer][1].length - + diffs[pointer - 1][1].length); + diffs[pointer + 1][1] = diffs[pointer - 1][1] + diffs[pointer + 1][1]; + diffs.splice(pointer - 1, 1); + changes = true; + } else if (diffs[pointer][1].substring(0, diffs[pointer + 1][1].length) == + diffs[pointer + 1][1]) { + // Shift the edit over the next equality. + diffs[pointer - 1][1] += diffs[pointer + 1][1]; + diffs[pointer][1] = + diffs[pointer][1].substring(diffs[pointer + 1][1].length) + + diffs[pointer + 1][1]; + diffs.splice(pointer + 1, 1); + changes = true; + } + } + pointer++; + } + // If shifts were made, the diff needs reordering and another shift sweep. + if (changes) { + this.diff_cleanupMerge(diffs); + } +}; + + +/** + * loc is a location in text1, compute and return the equivalent location in + * text2. + * e.g. 'The cat' vs 'The big cat', 1->1, 5->8 + * @param {!Array.} diffs Array of diff tuples. + * @param {number} loc Location within text1. + * @return {number} Location within text2. + */ +diff_match_patch.prototype.diff_xIndex = function(diffs, loc) { + var chars1 = 0; + var chars2 = 0; + var last_chars1 = 0; + var last_chars2 = 0; + var x; + for (x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_INSERT) { // Equality or deletion. + chars1 += diffs[x][1].length; + } + if (diffs[x][0] !== DIFF_DELETE) { // Equality or insertion. + chars2 += diffs[x][1].length; + } + if (chars1 > loc) { // Overshot the location. + break; + } + last_chars1 = chars1; + last_chars2 = chars2; + } + // Was the location was deleted? + if (diffs.length != x && diffs[x][0] === DIFF_DELETE) { + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); +}; + + +/** + * Convert a diff array into a pretty HTML report. + * @param {!Array.} diffs Array of diff tuples. + * @return {string} HTML representation. + */ +diff_match_patch.prototype.diff_prettyHtml = function(diffs) { + var html = []; + var pattern_amp = /&/g; + var pattern_lt = //g; + var pattern_para = /\n/g; + for (var x = 0; x < diffs.length; x++) { + var op = diffs[x][0]; // Operation (insert, delete, equal) + var data = diffs[x][1]; // Text of change. + var text = data.replace(pattern_amp, '&').replace(pattern_lt, '<') + .replace(pattern_gt, '>').replace(pattern_para, '¶
    '); + switch (op) { + case DIFF_INSERT: + html[x] = '' + text + ''; + break; + case DIFF_DELETE: + html[x] = '' + text + ''; + break; + case DIFF_EQUAL: + html[x] = '' + text + ''; + break; + } + } + return html.join(''); +}; + + +/** + * Compute and return the source text (all equalities and deletions). + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Source text. + */ +diff_match_patch.prototype.diff_text1 = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_INSERT) { + text[x] = diffs[x][1]; + } + } + return text.join(''); +}; + + +/** + * Compute and return the destination text (all equalities and insertions). + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Destination text. + */ +diff_match_patch.prototype.diff_text2 = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_DELETE) { + text[x] = diffs[x][1]; + } + } + return text.join(''); +}; + + +/** + * Compute the Levenshtein distance; the number of inserted, deleted or + * substituted characters. + * @param {!Array.} diffs Array of diff tuples. + * @return {number} Number of changes. + */ +diff_match_patch.prototype.diff_levenshtein = function(diffs) { + var levenshtein = 0; + var insertions = 0; + var deletions = 0; + for (var x = 0; x < diffs.length; x++) { + var op = diffs[x][0]; + var data = diffs[x][1]; + switch (op) { + case DIFF_INSERT: + insertions += data.length; + break; + case DIFF_DELETE: + deletions += data.length; + break; + case DIFF_EQUAL: + // A deletion and an insertion is one substitution. + levenshtein += Math.max(insertions, deletions); + insertions = 0; + deletions = 0; + break; + } + } + levenshtein += Math.max(insertions, deletions); + return levenshtein; +}; + + +/** + * Crush the diff into an encoded string which describes the operations + * required to transform text1 into text2. + * E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. + * Operations are tab-separated. Inserted text is escaped using %xx notation. + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Delta text. + */ +diff_match_patch.prototype.diff_toDelta = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + switch (diffs[x][0]) { + case DIFF_INSERT: + text[x] = '+' + encodeURI(diffs[x][1]); + break; + case DIFF_DELETE: + text[x] = '-' + diffs[x][1].length; + break; + case DIFF_EQUAL: + text[x] = '=' + diffs[x][1].length; + break; + } + } + return text.join('\t').replace(/%20/g, ' '); +}; + + +/** + * Given the original text1, and an encoded string which describes the + * operations required to transform text1 into text2, compute the full diff. + * @param {string} text1 Source string for the diff. + * @param {string} delta Delta text. + * @return {!Array.} Array of diff tuples. + * @throws {!Error} If invalid input. + */ +diff_match_patch.prototype.diff_fromDelta = function(text1, delta) { + var diffs = []; + var diffsLength = 0; // Keeping our own length var is faster in JS. + var pointer = 0; // Cursor in text1 + var tokens = delta.split(/\t/g); + for (var x = 0; x < tokens.length; x++) { + // Each token begins with a one character parameter which specifies the + // operation of this token (delete, insert, equality). + var param = tokens[x].substring(1); + switch (tokens[x].charAt(0)) { + case '+': + try { + diffs[diffsLength++] = + new diff_match_patch.Diff(DIFF_INSERT, decodeURI(param)); + } catch (ex) { + // Malformed URI sequence. + throw new Error('Illegal escape in diff_fromDelta: ' + param); + } + break; + case '-': + // Fall through. + case '=': + var n = parseInt(param, 10); + if (isNaN(n) || n < 0) { + throw new Error('Invalid number in diff_fromDelta: ' + param); + } + var text = text1.substring(pointer, pointer += n); + if (tokens[x].charAt(0) == '=') { + diffs[diffsLength++] = new diff_match_patch.Diff(DIFF_EQUAL, text); + } else { + diffs[diffsLength++] = new diff_match_patch.Diff(DIFF_DELETE, text); + } + break; + default: + // Blank tokens are ok (from a trailing \t). + // Anything else is an error. + if (tokens[x]) { + throw new Error('Invalid diff operation in diff_fromDelta: ' + + tokens[x]); + } + } + } + if (pointer != text1.length) { + throw new Error('Delta length (' + pointer + + ') does not equal source text length (' + text1.length + ').'); + } + return diffs; +}; + + +// MATCH FUNCTIONS + + +/** + * Locate the best instance of 'pattern' in 'text' near 'loc'. + * @param {string} text The text to search. + * @param {string} pattern The pattern to search for. + * @param {number} loc The location to search around. + * @return {number} Best match index or -1. + */ +diff_match_patch.prototype.match_main = function(text, pattern, loc) { + // Check for null inputs. + if (text == null || pattern == null || loc == null) { + throw new Error('Null input. (match_main)'); + } + + loc = Math.max(0, Math.min(loc, text.length)); + if (text == pattern) { + // Shortcut (potentially not guaranteed by the algorithm) + return 0; + } else if (!text.length) { + // Nothing to match. + return -1; + } else if (text.substring(loc, loc + pattern.length) == pattern) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + return this.match_bitap_(text, pattern, loc); + } +}; + + +/** + * Locate the best instance of 'pattern' in 'text' near 'loc' using the + * Bitap algorithm. + * @param {string} text The text to search. + * @param {string} pattern The pattern to search for. + * @param {number} loc The location to search around. + * @return {number} Best match index or -1. + * @private + */ +diff_match_patch.prototype.match_bitap_ = function(text, pattern, loc) { + if (pattern.length > this.Match_MaxBits) { + throw new Error('Pattern too long for this browser.'); + } + + // Initialise the alphabet. + var s = this.match_alphabet_(pattern); + + var dmp = this; // 'this' becomes 'window' in a closure. + + /** + * Compute and return the score for a match with e errors and x location. + * Accesses loc and pattern through being a closure. + * @param {number} e Number of errors in match. + * @param {number} x Location of match. + * @return {number} Overall score for match (0.0 = good, 1.0 = bad). + * @private + */ + function match_bitapScore_(e, x) { + var accuracy = e / pattern.length; + var proximity = Math.abs(loc - x); + if (!dmp.Match_Distance) { + // Dodge divide by zero error. + return proximity ? 1.0 : accuracy; + } + return accuracy + (proximity / dmp.Match_Distance); + } + + // Highest score beyond which we give up. + var score_threshold = this.Match_Threshold; + // Is there a nearby exact match? (speedup) + var best_loc = text.indexOf(pattern, loc); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore_(0, best_loc), score_threshold); + // What about in the other direction? (speedup) + best_loc = text.lastIndexOf(pattern, loc + pattern.length); + if (best_loc != -1) { + score_threshold = + Math.min(match_bitapScore_(0, best_loc), score_threshold); + } + } + + // Initialise the bit arrays. + var matchmask = 1 << (pattern.length - 1); + best_loc = -1; + + var bin_min, bin_mid; + var bin_max = pattern.length + text.length; + var last_rd; + for (var d = 0; d < pattern.length; d++) { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at this + // error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (match_bitapScore_(d, loc + bin_mid) <= score_threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = Math.floor((bin_max - bin_min) / 2 + bin_min); + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = Math.max(1, loc - bin_mid + 1); + var finish = Math.min(loc + bin_mid, text.length) + pattern.length; + + var rd = Array(finish + 2); + rd[finish + 1] = (1 << d) - 1; + for (var j = finish; j >= start; j--) { + // The alphabet (s) is a sparse hash, so the following line generates + // warnings. + var charMatch = s[text.charAt(j - 1)]; + if (d === 0) { // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & charMatch; + } else { // Subsequent passes: fuzzy match. + rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) | + (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | + last_rd[j + 1]; + } + if (rd[j] & matchmask) { + var score = match_bitapScore_(d, j - 1); + // This match will almost certainly be better than any existing match. + // But check anyway. + if (score <= score_threshold) { + // Told you so. + score_threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = Math.max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + // No hope for a (better) match at greater error levels. + if (match_bitapScore_(d + 1, loc) > score_threshold) { + break; + } + last_rd = rd; + } + return best_loc; +}; + + +/** + * Initialise the alphabet for the Bitap algorithm. + * @param {string} pattern The text to encode. + * @return {!Object} Hash of character locations. + * @private + */ +diff_match_patch.prototype.match_alphabet_ = function(pattern) { + var s = {}; + for (var i = 0; i < pattern.length; i++) { + s[pattern.charAt(i)] = 0; + } + for (var i = 0; i < pattern.length; i++) { + s[pattern.charAt(i)] |= 1 << (pattern.length - i - 1); + } + return s; +}; + + +// PATCH FUNCTIONS + + +/** + * Increase the context until it is unique, + * but don't let the pattern expand beyond Match_MaxBits. + * @param {!diff_match_patch.patch_obj} patch The patch to grow. + * @param {string} text Source text. + * @private + */ +diff_match_patch.prototype.patch_addContext_ = function(patch, text) { + if (text.length == 0) { + return; + } + if (patch.start2 === null) { + throw Error('patch not initialized'); + } + var pattern = text.substring(patch.start2, patch.start2 + patch.length1); + var padding = 0; + + // Look for the first and last matches of pattern in text. If two different + // matches are found, increase the pattern length. + while (text.indexOf(pattern) != text.lastIndexOf(pattern) && + pattern.length < this.Match_MaxBits - this.Patch_Margin - + this.Patch_Margin) { + padding += this.Patch_Margin; + pattern = text.substring(patch.start2 - padding, + patch.start2 + patch.length1 + padding); + } + // Add one chunk for good luck. + padding += this.Patch_Margin; + + // Add the prefix. + var prefix = text.substring(patch.start2 - padding, patch.start2); + if (prefix) { + patch.diffs.unshift(new diff_match_patch.Diff(DIFF_EQUAL, prefix)); + } + // Add the suffix. + var suffix = text.substring(patch.start2 + patch.length1, + patch.start2 + patch.length1 + padding); + if (suffix) { + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, suffix)); + } + + // Roll back the start points. + patch.start1 -= prefix.length; + patch.start2 -= prefix.length; + // Extend the lengths. + patch.length1 += prefix.length + suffix.length; + patch.length2 += prefix.length + suffix.length; +}; + + +/** + * Compute a list of patches to turn text1 into text2. + * Use diffs if provided, otherwise compute it ourselves. + * There are four ways to call this function, depending on what data is + * available to the caller: + * Method 1: + * a = text1, b = text2 + * Method 2: + * a = diffs + * Method 3 (optimal): + * a = text1, b = diffs + * Method 4 (deprecated, use method 3): + * a = text1, b = text2, c = diffs + * + * @param {string|!Array.} a text1 (methods 1,3,4) or + * Array of diff tuples for text1 to text2 (method 2). + * @param {string|!Array.=} opt_b text2 (methods 1,4) or + * Array of diff tuples for text1 to text2 (method 3) or undefined (method 2). + * @param {string|!Array.=} opt_c Array of diff tuples + * for text1 to text2 (method 4) or undefined (methods 1,2,3). + * @return {!Array.} Array of Patch objects. + */ +diff_match_patch.prototype.patch_make = function(a, opt_b, opt_c) { + var text1, diffs; + if (typeof a == 'string' && typeof opt_b == 'string' && + typeof opt_c == 'undefined') { + // Method 1: text1, text2 + // Compute diffs from text1 and text2. + text1 = /** @type {string} */(a); + diffs = this.diff_main(text1, /** @type {string} */(opt_b), true); + if (diffs.length > 2) { + this.diff_cleanupSemantic(diffs); + this.diff_cleanupEfficiency(diffs); + } + } else if (a && typeof a == 'object' && typeof opt_b == 'undefined' && + typeof opt_c == 'undefined') { + // Method 2: diffs + // Compute text1 from diffs. + diffs = /** @type {!Array.} */(a); + text1 = this.diff_text1(diffs); + } else if (typeof a == 'string' && opt_b && typeof opt_b == 'object' && + typeof opt_c == 'undefined') { + // Method 3: text1, diffs + text1 = /** @type {string} */(a); + diffs = /** @type {!Array.} */(opt_b); + } else if (typeof a == 'string' && typeof opt_b == 'string' && + opt_c && typeof opt_c == 'object') { + // Method 4: text1, text2, diffs + // text2 is not used. + text1 = /** @type {string} */(a); + diffs = /** @type {!Array.} */(opt_c); + } else { + throw new Error('Unknown call format to patch_make.'); + } + + if (diffs.length === 0) { + return []; // Get rid of the null case. + } + var patches = []; + var patch = new diff_match_patch.patch_obj(); + var patchDiffLength = 0; // Keeping our own length var is faster in JS. + var char_count1 = 0; // Number of characters into the text1 string. + var char_count2 = 0; // Number of characters into the text2 string. + // Start with text1 (prepatch_text) and apply the diffs until we arrive at + // text2 (postpatch_text). We recreate the patches one by one to determine + // context info. + var prepatch_text = text1; + var postpatch_text = text1; + for (var x = 0; x < diffs.length; x++) { + var diff_type = diffs[x][0]; + var diff_text = diffs[x][1]; + + if (!patchDiffLength && diff_type !== DIFF_EQUAL) { + // A new patch starts here. + patch.start1 = char_count1; + patch.start2 = char_count2; + } + + switch (diff_type) { + case DIFF_INSERT: + patch.diffs[patchDiffLength++] = diffs[x]; + patch.length2 += diff_text.length; + postpatch_text = postpatch_text.substring(0, char_count2) + diff_text + + postpatch_text.substring(char_count2); + break; + case DIFF_DELETE: + patch.length1 += diff_text.length; + patch.diffs[patchDiffLength++] = diffs[x]; + postpatch_text = postpatch_text.substring(0, char_count2) + + postpatch_text.substring(char_count2 + + diff_text.length); + break; + case DIFF_EQUAL: + if (diff_text.length <= 2 * this.Patch_Margin && + patchDiffLength && diffs.length != x + 1) { + // Small equality inside a patch. + patch.diffs[patchDiffLength++] = diffs[x]; + patch.length1 += diff_text.length; + patch.length2 += diff_text.length; + } else if (diff_text.length >= 2 * this.Patch_Margin) { + // Time for a new patch. + if (patchDiffLength) { + this.patch_addContext_(patch, prepatch_text); + patches.push(patch); + patch = new diff_match_patch.patch_obj(); + patchDiffLength = 0; + // Unlike Unidiff, our patch lists have a rolling context. + // https://github.com/google/diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + prepatch_text = postpatch_text; + char_count1 = char_count2; + } + } + break; + } + + // Update the current character count. + if (diff_type !== DIFF_INSERT) { + char_count1 += diff_text.length; + } + if (diff_type !== DIFF_DELETE) { + char_count2 += diff_text.length; + } + } + // Pick up the leftover patch if not empty. + if (patchDiffLength) { + this.patch_addContext_(patch, prepatch_text); + patches.push(patch); + } + + return patches; +}; + + +/** + * Given an array of patches, return another array that is identical. + * @param {!Array.} patches Array of Patch objects. + * @return {!Array.} Array of Patch objects. + */ +diff_match_patch.prototype.patch_deepCopy = function(patches) { + // Making deep copies is hard in JavaScript. + var patchesCopy = []; + for (var x = 0; x < patches.length; x++) { + var patch = patches[x]; + var patchCopy = new diff_match_patch.patch_obj(); + patchCopy.diffs = []; + for (var y = 0; y < patch.diffs.length; y++) { + patchCopy.diffs[y] = + new diff_match_patch.Diff(patch.diffs[y][0], patch.diffs[y][1]); + } + patchCopy.start1 = patch.start1; + patchCopy.start2 = patch.start2; + patchCopy.length1 = patch.length1; + patchCopy.length2 = patch.length2; + patchesCopy[x] = patchCopy; + } + return patchesCopy; +}; + + +/** + * Merge a set of patches onto the text. Return a patched text, as well + * as a list of true/false values indicating which patches were applied. + * @param {!Array.} patches Array of Patch objects. + * @param {string} text Old text. + * @return {!Array.>} Two element Array, containing the + * new text and an array of boolean values. + */ +diff_match_patch.prototype.patch_apply = function(patches, text) { + if (patches.length == 0) { + return [text, []]; + } + + // Deep copy the patches so that no changes are made to originals. + patches = this.patch_deepCopy(patches); + + var nullPadding = this.patch_addPadding(patches); + text = nullPadding + text + nullPadding; + + this.patch_splitMax(patches); + // delta keeps track of the offset between the expected and actual location + // of the previous patch. If there are patches expected at positions 10 and + // 20, but the first patch was found at 12, delta is 2 and the second patch + // has an effective expected position of 22. + var delta = 0; + var results = []; + for (var x = 0; x < patches.length; x++) { + var expected_loc = patches[x].start2 + delta; + var text1 = this.diff_text1(patches[x].diffs); + var start_loc; + var end_loc = -1; + if (text1.length > this.Match_MaxBits) { + // patch_splitMax will only provide an oversized pattern in the case of + // a monster delete. + start_loc = this.match_main(text, text1.substring(0, this.Match_MaxBits), + expected_loc); + if (start_loc != -1) { + end_loc = this.match_main(text, + text1.substring(text1.length - this.Match_MaxBits), + expected_loc + text1.length - this.Match_MaxBits); + if (end_loc == -1 || start_loc >= end_loc) { + // Can't find valid trailing context. Drop this patch. + start_loc = -1; + } + } + } else { + start_loc = this.match_main(text, text1, expected_loc); + } + if (start_loc == -1) { + // No match found. :( + results[x] = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= patches[x].length2 - patches[x].length1; + } else { + // Found a match. :) + results[x] = true; + delta = start_loc - expected_loc; + var text2; + if (end_loc == -1) { + text2 = text.substring(start_loc, start_loc + text1.length); + } else { + text2 = text.substring(start_loc, end_loc + this.Match_MaxBits); + } + if (text1 == text2) { + // Perfect match, just shove the replacement text in. + text = text.substring(0, start_loc) + + this.diff_text2(patches[x].diffs) + + text.substring(start_loc + text1.length); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + var diffs = this.diff_main(text1, text2, false); + if (text1.length > this.Match_MaxBits && + this.diff_levenshtein(diffs) / text1.length > + this.Patch_DeleteThreshold) { + // The end points match, but the content is unacceptably bad. + results[x] = false; + } else { + this.diff_cleanupSemanticLossless(diffs); + var index1 = 0; + var index2; + for (var y = 0; y < patches[x].diffs.length; y++) { + var mod = patches[x].diffs[y]; + if (mod[0] !== DIFF_EQUAL) { + index2 = this.diff_xIndex(diffs, index1); + } + if (mod[0] === DIFF_INSERT) { // Insertion + text = text.substring(0, start_loc + index2) + mod[1] + + text.substring(start_loc + index2); + } else if (mod[0] === DIFF_DELETE) { // Deletion + text = text.substring(0, start_loc + index2) + + text.substring(start_loc + this.diff_xIndex(diffs, + index1 + mod[1].length)); + } + if (mod[0] !== DIFF_DELETE) { + index1 += mod[1].length; + } + } + } + } + } + } + // Strip the padding off. + text = text.substring(nullPadding.length, text.length - nullPadding.length); + return [text, results]; +}; + + +/** + * Add some padding on text start and end so that edges can match something. + * Intended to be called only from within patch_apply. + * @param {!Array.} patches Array of Patch objects. + * @return {string} The padding string added to each side. + */ +diff_match_patch.prototype.patch_addPadding = function(patches) { + var paddingLength = this.Patch_Margin; + var nullPadding = ''; + for (var x = 1; x <= paddingLength; x++) { + nullPadding += String.fromCharCode(x); + } + + // Bump all the patches forward. + for (var x = 0; x < patches.length; x++) { + patches[x].start1 += paddingLength; + patches[x].start2 += paddingLength; + } + + // Add some padding on start of first diff. + var patch = patches[0]; + var diffs = patch.diffs; + if (diffs.length == 0 || diffs[0][0] != DIFF_EQUAL) { + // Add nullPadding equality. + diffs.unshift(new diff_match_patch.Diff(DIFF_EQUAL, nullPadding)); + patch.start1 -= paddingLength; // Should be 0. + patch.start2 -= paddingLength; // Should be 0. + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs[0][1].length) { + // Grow first equality. + var extraLength = paddingLength - diffs[0][1].length; + diffs[0][1] = nullPadding.substring(diffs[0][1].length) + diffs[0][1]; + patch.start1 -= extraLength; + patch.start2 -= extraLength; + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + // Add some padding on end of last diff. + patch = patches[patches.length - 1]; + diffs = patch.diffs; + if (diffs.length == 0 || diffs[diffs.length - 1][0] != DIFF_EQUAL) { + // Add nullPadding equality. + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, nullPadding)); + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs[diffs.length - 1][1].length) { + // Grow last equality. + var extraLength = paddingLength - diffs[diffs.length - 1][1].length; + diffs[diffs.length - 1][1] += nullPadding.substring(0, extraLength); + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + return nullPadding; +}; + + +/** + * Look through the patches and break up any which are longer than the maximum + * limit of the match algorithm. + * Intended to be called only from within patch_apply. + * @param {!Array.} patches Array of Patch objects. + */ +diff_match_patch.prototype.patch_splitMax = function(patches) { + var patch_size = this.Match_MaxBits; + for (var x = 0; x < patches.length; x++) { + if (patches[x].length1 <= patch_size) { + continue; + } + var bigpatch = patches[x]; + // Remove the big old patch. + patches.splice(x--, 1); + var start1 = bigpatch.start1; + var start2 = bigpatch.start2; + var precontext = ''; + while (bigpatch.diffs.length !== 0) { + // Create one of several smaller patches. + var patch = new diff_match_patch.patch_obj(); + var empty = true; + patch.start1 = start1 - precontext.length; + patch.start2 = start2 - precontext.length; + if (precontext !== '') { + patch.length1 = patch.length2 = precontext.length; + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, precontext)); + } + while (bigpatch.diffs.length !== 0 && + patch.length1 < patch_size - this.Patch_Margin) { + var diff_type = bigpatch.diffs[0][0]; + var diff_text = bigpatch.diffs[0][1]; + if (diff_type === DIFF_INSERT) { + // Insertions are harmless. + patch.length2 += diff_text.length; + start2 += diff_text.length; + patch.diffs.push(bigpatch.diffs.shift()); + empty = false; + } else if (diff_type === DIFF_DELETE && patch.diffs.length == 1 && + patch.diffs[0][0] == DIFF_EQUAL && + diff_text.length > 2 * patch_size) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.length; + start1 += diff_text.length; + empty = false; + patch.diffs.push(new diff_match_patch.Diff(diff_type, diff_text)); + bigpatch.diffs.shift(); + } else { + // Deletion or equality. Only take as much as we can stomach. + diff_text = diff_text.substring(0, + patch_size - patch.length1 - this.Patch_Margin); + patch.length1 += diff_text.length; + start1 += diff_text.length; + if (diff_type === DIFF_EQUAL) { + patch.length2 += diff_text.length; + start2 += diff_text.length; + } else { + empty = false; + } + patch.diffs.push(new diff_match_patch.Diff(diff_type, diff_text)); + if (diff_text == bigpatch.diffs[0][1]) { + bigpatch.diffs.shift(); + } else { + bigpatch.diffs[0][1] = + bigpatch.diffs[0][1].substring(diff_text.length); + } + } + } + // Compute the head context for the next patch. + precontext = this.diff_text2(patch.diffs); + precontext = + precontext.substring(precontext.length - this.Patch_Margin); + // Append the end context for this patch. + var postcontext = this.diff_text1(bigpatch.diffs) + .substring(0, this.Patch_Margin); + if (postcontext !== '') { + patch.length1 += postcontext.length; + patch.length2 += postcontext.length; + if (patch.diffs.length !== 0 && + patch.diffs[patch.diffs.length - 1][0] === DIFF_EQUAL) { + patch.diffs[patch.diffs.length - 1][1] += postcontext; + } else { + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, postcontext)); + } + } + if (!empty) { + patches.splice(++x, 0, patch); + } + } + } +}; + + +/** + * Take a list of patches and return a textual representation. + * @param {!Array.} patches Array of Patch objects. + * @return {string} Text representation of patches. + */ +diff_match_patch.prototype.patch_toText = function(patches) { + var text = []; + for (var x = 0; x < patches.length; x++) { + text[x] = patches[x]; + } + return text.join(''); +}; + + +/** + * Parse a textual representation of patches and return a list of Patch objects. + * @param {string} textline Text representation of patches. + * @return {!Array.} Array of Patch objects. + * @throws {!Error} If invalid input. + */ +diff_match_patch.prototype.patch_fromText = function(textline) { + var patches = []; + if (!textline) { + return patches; + } + var text = textline.split('\n'); + var textPointer = 0; + var patchHeader = /^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$/; + while (textPointer < text.length) { + var m = text[textPointer].match(patchHeader); + if (!m) { + throw new Error('Invalid patch string: ' + text[textPointer]); + } + var patch = new diff_match_patch.patch_obj(); + patches.push(patch); + patch.start1 = parseInt(m[1], 10); + if (m[2] === '') { + patch.start1--; + patch.length1 = 1; + } else if (m[2] == '0') { + patch.length1 = 0; + } else { + patch.start1--; + patch.length1 = parseInt(m[2], 10); + } + + patch.start2 = parseInt(m[3], 10); + if (m[4] === '') { + patch.start2--; + patch.length2 = 1; + } else if (m[4] == '0') { + patch.length2 = 0; + } else { + patch.start2--; + patch.length2 = parseInt(m[4], 10); + } + textPointer++; + + while (textPointer < text.length) { + var sign = text[textPointer].charAt(0); + try { + var line = decodeURI(text[textPointer].substring(1)); + } catch (ex) { + // Malformed URI sequence. + throw new Error('Illegal escape in patch_fromText: ' + line); + } + if (sign == '-') { + // Deletion. + patch.diffs.push(new diff_match_patch.Diff(DIFF_DELETE, line)); + } else if (sign == '+') { + // Insertion. + patch.diffs.push(new diff_match_patch.Diff(DIFF_INSERT, line)); + } else if (sign == ' ') { + // Minor equality. + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, line)); + } else if (sign == '@') { + // Start of next patch. + break; + } else if (sign === '') { + // Blank line? Whatever. + } else { + // WTF? + throw new Error('Invalid patch mode "' + sign + '" in: ' + line); + } + textPointer++; + } + } + return patches; +}; + + +/** + * Class representing one patch operation. + * @constructor + */ +diff_match_patch.patch_obj = function() { + /** @type {!Array.} */ + this.diffs = []; + /** @type {?number} */ + this.start1 = null; + /** @type {?number} */ + this.start2 = null; + /** @type {number} */ + this.length1 = 0; + /** @type {number} */ + this.length2 = 0; +}; + + +/** + * Emulate GNU diff's format. + * Header: @@ -382,8 +481,9 @@ + * Indices are printed as 1-based, not 0-based. + * @return {string} The GNU diff string. + */ +diff_match_patch.patch_obj.prototype.toString = function() { + var coords1, coords2; + if (this.length1 === 0) { + coords1 = this.start1 + ',0'; + } else if (this.length1 == 1) { + coords1 = this.start1 + 1; + } else { + coords1 = (this.start1 + 1) + ',' + this.length1; + } + if (this.length2 === 0) { + coords2 = this.start2 + ',0'; + } else if (this.length2 == 1) { + coords2 = this.start2 + 1; + } else { + coords2 = (this.start2 + 1) + ',' + this.length2; + } + var text = ['@@ -' + coords1 + ' +' + coords2 + ' @@\n']; + var op; + // Escape the body of the patch with %xx notation. + for (var x = 0; x < this.diffs.length; x++) { + switch (this.diffs[x][0]) { + case DIFF_INSERT: + op = '+'; + break; + case DIFF_DELETE: + op = '-'; + break; + case DIFF_EQUAL: + op = ' '; + break; + } + text[x + 1] = op + encodeURI(this.diffs[x][1]) + '\n'; + } + return text.join('').replace(/%20/g, ' '); +}; + +// CLOSURE:begin_strip +// Lines below here will not be included in the Closure-compatible library. + +// Export these global variables so that they survive Google's JS compiler. +// In a browser, 'this' will be 'window'. +// Users of node.js should 'require' the uncompressed version since Google's +// JS compiler may break the following exports for non-browser environments. +/** @suppress {globalThis} */ +this['diff_match_patch'] = diff_match_patch; +/** @suppress {globalThis} */ +this['DIFF_DELETE'] = DIFF_DELETE; +/** @suppress {globalThis} */ +this['DIFF_INSERT'] = DIFF_INSERT; +/** @suppress {globalThis} */ +this['DIFF_EQUAL'] = DIFF_EQUAL; From 55ccf083c836043c7c242448762571494d41a75d Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:32:36 -0500 Subject: [PATCH 19/44] Add debug diff view --- background.js | 11 ++++++++++- options/options.html | 6 ++++++ options/options.js | 12 +++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/background.js b/background.js index b5667e6..58a5de5 100644 --- a/background.js +++ b/background.js @@ -33,6 +33,7 @@ let userTheme = 'auto'; let currentTheme = 'light'; let detectSystemTheme; let errorPending = false; +let showDebugTab = false; const ERROR_NOTIFICATION_ID = 'sortana-error'; function normalizeRules(rules) { @@ -262,12 +263,16 @@ async function processMessage(id) { try { const full = await messenger.messages.getFull(id); let text = buildEmailText(full); + const originalText = text; if (tokenReduction && maxTokens > 0) { const limit = Math.floor(maxTokens * 0.9); if (text.length > limit) { text = text.slice(0, limit); } } + if (showDebugTab) { + await storage.local.set({ lastFullText: originalText, lastPromptText: text }); + } let hdr; let currentTags = []; let alreadyRead = false; @@ -425,7 +430,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -440,6 +445,7 @@ async function clearCacheForMessages(idsInput) { maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; } errorPending = store.errorPending === true; + showDebugTab = store.showDebugTab === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { Object.assign(timingStats, savedStats.classifyStats); @@ -494,6 +500,9 @@ async function clearCacheForMessages(idsInput) { tokenReduction = changes.tokenReduction.newValue === true; logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction); } + if (changes.showDebugTab) { + showDebugTab = changes.showDebugTab.newValue === true; + } if (changes.errorPending) { errorPending = changes.errorPending.newValue === true; updateActionIcon(); diff --git a/options/options.html b/options/options.html index d14ce99..3618a20 100644 --- a/options/options.html +++ b/options/options.html @@ -31,6 +31,10 @@ .tag { --bulma-tag-h: 318; } + #diff-display { + white-space: pre-wrap; + font-family: monospace; + } @@ -286,9 +290,11 @@ Debug
    
    +                
    + diff --git a/options/options.js b/options/options.js index ebd7497..3a09d37 100644 --- a/options/options.js +++ b/options/options.js @@ -21,7 +21,9 @@ document.addEventListener('DOMContentLoaded', async () => { 'aiCache', 'theme', 'showDebugTab', - 'lastPayload' + 'lastPayload', + 'lastFullText', + 'lastPromptText' ]); const tabButtons = document.querySelectorAll('#main-tabs li'); const tabs = document.querySelectorAll('.tab-content'); @@ -67,9 +69,17 @@ document.addEventListener('DOMContentLoaded', async () => { await applyTheme(themeSelect.value); const payloadDisplay = document.getElementById('payload-display'); + const diffDisplay = document.getElementById('diff-display'); if (defaults.lastPayload) { payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2); } + if (defaults.lastFullText && defaults.lastPromptText && diff_match_patch) { + const dmp = new diff_match_patch(); + dmp.Diff_EditCost = 4; + const diffs = dmp.diff_main(defaults.lastFullText, defaults.lastPromptText); + dmp.diff_cleanupEfficiency(diffs); + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + } themeSelect.addEventListener('change', async () => { markDirty(); await applyTheme(themeSelect.value); From 841a697c69a9204bcaf7c22c30be8fd5259837f5 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:46:44 -0500 Subject: [PATCH 20/44] Update debug tab with live refresh --- README.md | 2 +- options/options.js | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 64fbf26..82649d0 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ message meets a specified criterion. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. -- **Debug tab** – view the last request payload sent to the AI service. +- **Debug tab** – view the last request payload and message diff with live updates. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/options/options.js b/options/options.js index 3a09d37..3402dcf 100644 --- a/options/options.js +++ b/options/options.js @@ -70,13 +70,18 @@ document.addEventListener('DOMContentLoaded', async () => { await applyTheme(themeSelect.value); const payloadDisplay = document.getElementById('payload-display'); const diffDisplay = document.getElementById('diff-display'); - if (defaults.lastPayload) { - payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2); + + let lastFullText = defaults.lastFullText || ''; + let lastPromptText = defaults.lastPromptText || ''; + let lastPayload = defaults.lastPayload ? JSON.stringify(defaults.lastPayload, null, 2) : ''; + + if (lastPayload) { + payloadDisplay.textContent = lastPayload; } - if (defaults.lastFullText && defaults.lastPromptText && diff_match_patch) { + if (lastFullText && lastPromptText && diff_match_patch) { const dmp = new diff_match_patch(); dmp.Diff_EditCost = 4; - const diffs = dmp.diff_main(defaults.lastFullText, defaults.lastPromptText); + const diffs = dmp.diff_main(lastFullText, lastPromptText); dmp.diff_cleanupEfficiency(diffs); diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); } @@ -729,6 +734,30 @@ document.addEventListener('DOMContentLoaded', async () => { } catch { cacheCountEl.textContent = '?'; } + + try { + if (debugTabToggle.checked) { + const latest = await storage.local.get(['lastPayload', 'lastFullText', 'lastPromptText']); + const payloadStr = latest.lastPayload ? JSON.stringify(latest.lastPayload, null, 2) : ''; + if (payloadStr !== lastPayload) { + lastPayload = payloadStr; + payloadDisplay.textContent = payloadStr; + } + if (latest.lastFullText !== lastFullText || latest.lastPromptText !== lastPromptText) { + lastFullText = latest.lastFullText || ''; + lastPromptText = latest.lastPromptText || ''; + if (lastFullText && lastPromptText && diff_match_patch) { + const dmp = new diff_match_patch(); + dmp.Diff_EditCost = 4; + const diffs = dmp.diff_main(lastFullText, lastPromptText); + dmp.diff_cleanupEfficiency(diffs); + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + } else { + diffDisplay.innerHTML = ''; + } + } + } + } catch {} } refreshMaintenance(); From bcac4ad01709c85002e2f8123406bacffcf1d4cd Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 21:58:37 -0500 Subject: [PATCH 21/44] Improve debug tab layout --- options/options.html | 15 +++++++++------ options/options.js | 22 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/options/options.html b/options/options.html index 3618a20..ddc5ee0 100644 --- a/options/options.html +++ b/options/options.html @@ -154,6 +154,11 @@ Aggressive token reduction +
    + +
    @@ -220,11 +225,6 @@
    -
    - -
    @@ -290,7 +290,10 @@ Debug
    
    -                
    + diff --git a/options/options.js b/options/options.js index 3402dcf..d881d6a 100644 --- a/options/options.js +++ b/options/options.js @@ -70,6 +70,7 @@ document.addEventListener('DOMContentLoaded', async () => { await applyTheme(themeSelect.value); const payloadDisplay = document.getElementById('payload-display'); const diffDisplay = document.getElementById('diff-display'); + const diffContainer = document.getElementById('diff-container'); let lastFullText = defaults.lastFullText || ''; let lastPromptText = defaults.lastPromptText || ''; @@ -83,7 +84,16 @@ document.addEventListener('DOMContentLoaded', async () => { dmp.Diff_EditCost = 4; const diffs = dmp.diff_main(lastFullText, lastPromptText); dmp.diff_cleanupEfficiency(diffs); - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + const hasDiff = diffs.some(d => d[0] !== 0); + if (hasDiff) { + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + diffContainer.classList.remove('is-hidden'); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + } + } else { + diffContainer.classList.add('is-hidden'); } themeSelect.addEventListener('change', async () => { markDirty(); @@ -751,9 +761,17 @@ document.addEventListener('DOMContentLoaded', async () => { dmp.Diff_EditCost = 4; const diffs = dmp.diff_main(lastFullText, lastPromptText); dmp.diff_cleanupEfficiency(diffs); - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + const hasDiff = diffs.some(d => d[0] !== 0); + if (hasDiff) { + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + diffContainer.classList.remove('is-hidden'); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + } } else { diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); } } } From 9cad2674e3841fbc40d237f7233fbd6784f61760 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 22:44:04 -0500 Subject: [PATCH 22/44] Capture raw message text for debug --- README.md | 2 +- background.js | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 82649d0..57a0285 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ message meets a specified criterion. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. -- **Debug tab** – view the last request payload and message diff with live updates. +- **Debug tab** – view the last request payload and a diff between the unaltered message text and the final prompt. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/background.js b/background.js index 58a5de5..fc585ff 100644 --- a/background.js +++ b/background.js @@ -210,17 +210,38 @@ function collectText(part, bodyParts, attachments) { } } -function buildEmailText(full) { +function collectRawText(part, bodyParts, attachments) { + if (part.parts && part.parts.length) { + for (const p of part.parts) collectRawText(p, bodyParts, attachments); + return; + } + const ct = (part.contentType || "text/plain").toLowerCase(); + const cd = (part.headers?.["content-disposition"]?.[0] || "").toLowerCase(); + const body = String(part.body || ""); + if (cd.includes("attachment") || !ct.startsWith("text/")) { + const nameMatch = /filename\s*=\s*"?([^";]+)/i.exec(cd) || /name\s*=\s*"?([^";]+)/i.exec(part.headers?.["content-type"]?.[0] || ""); + const name = nameMatch ? nameMatch[1] : ""; + attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`); + } else if (ct.startsWith("text/html")) { + const doc = new DOMParser().parseFromString(body, 'text/html'); + bodyParts.push(doc.body.textContent || ""); + } else { + bodyParts.push(body); + } +} + +function buildEmailText(full, applyTransforms = true) { const bodyParts = []; const attachments = []; - collectText(full, bodyParts, attachments); + const collect = applyTransforms ? collectText : collectRawText; + collect(full, bodyParts, attachments); const headers = Object.entries(full.headers || {}) .map(([k, v]) => `${k}: ${v.join(' ')}`) .join('\n'); const attachInfo = `Attachments: ${attachments.length}` + (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : ""); let combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); - if (tokenReduction) { + if (applyTransforms && tokenReduction) { const seen = new Set(); combined = combined.split('\n').filter(l => { if (seen.has(l)) return false; @@ -228,7 +249,7 @@ function buildEmailText(full) { return true; }).join('\n'); } - return sanitizeString(combined); + return applyTransforms ? sanitizeString(combined) : combined; } function updateTimingStats(elapsed) { @@ -262,8 +283,8 @@ async function processMessage(id) { updateActionIcon(); try { const full = await messenger.messages.getFull(id); + const originalText = buildEmailText(full, false); let text = buildEmailText(full); - const originalText = text; if (tokenReduction && maxTokens > 0) { const limit = Math.floor(maxTokens * 0.9); if (text.length > limit) { From c622c07c66a6936c4c8d3bb039b1f0bba4692384 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 18 Aug 2025 19:39:35 -0500 Subject: [PATCH 23/44] Supporting v140+ After testing in Betterbird v140, updating manifest. --- manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index a3c9f7c..e7cb9d8 100644 --- a/manifest.json +++ b/manifest.json @@ -1,13 +1,13 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.1.2", + "version": "2.2.0", "default_locale": "en-US", "applications": { "gecko": { "id": "ai-filter@jordanwages", "strict_min_version": "128.0", - "strict_max_version": "139.*" + "strict_max_version": "140.*" } }, "icons": { From 0f2f148b71913b89489cf72ab6bc0efc263c1064 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 20:45:31 -0600 Subject: [PATCH 24/44] Normalize completions endpoint base --- AGENTS.md | 5 ++++- README.md | 11 ++++++----- modules/AiClassifier.js | 41 +++++++++++++++++++++++++++++++++++++---- options/options.html | 1 + options/options.js | 18 ++++++++++++++++-- 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 401f962..e2a1696 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,10 @@ This file provides guidelines for codex agents contributing to the Sortana proje There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. Do **not** run the `ps1` build script or the SVG processing script. +## Endpoint Notes + +Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. + ## Documentation Additional documentation exists outside this repository. @@ -73,4 +77,3 @@ Toolbar and menu icons reside under `resources/img` and are provided in 16, 32 and 64 pixel variants. When changing these icons, pass a dictionary mapping the sizes to the paths in `browserAction.setIcon` or `messageDisplayAction.setIcon`. Use `resources/svg2img.ps1` to regenerate PNGs from the SVG sources. - diff --git a/README.md b/README.md index 57a0285..2f8f204 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,13 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filter rule. It allows you to classify email messages by sending their contents to a configurable -HTTP endpoint. The endpoint should respond with JSON indicating whether the -message meets a specified criterion. +HTTP endpoint. Sortana uses the `/v1/completions` API; the options page stores a base +URL and appends `/v1/completions` when sending requests. The endpoint should respond +with JSON indicating whether the message meets a specified criterion. ## Features -- **Configurable endpoint** – set the classification service URL on the options page. +- **Configurable endpoint** – set the classification service base URL on the options page. - **Prompt templates** – choose between several model formats or provide your own custom template. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. @@ -72,7 +73,8 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e ## Usage -1. Open the add-on's options and set the URL of your classification service. +1. Open the add-on's options and set the base URL of your classification service + (Sortana will append `/v1/completions`). 2. Use the **Classification Rules** section to add a criterion and optional actions such as tagging, moving, copying, forwarding, replying, deleting or archiving a message when it matches. Drag rules to @@ -158,4 +160,3 @@ how Thunderbird's WebExtension and experiment APIs can be extended. Their code provided invaluable guidance during development. - Icons from [cc0-icons.jonh.eu](https://cc0-icons.jonh.eu/) are used under the CC0 license. - diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 8313654..f5a3bff 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -15,6 +15,8 @@ try { Services = undefined; } +const COMPLETIONS_PATH = "/v1/completions"; + const SYSTEM_PREFIX = `You are an email-classification assistant. Read the email below and the classification criterion provided by the user. `; @@ -28,7 +30,8 @@ Return ONLY a JSON object on a single line of the form: Do not add any other keys, text, or formatting.`; -let gEndpoint = "http://127.0.0.1:5000/v1/classify"; +let gEndpointBase = "http://127.0.0.1:5000"; +let gEndpoint = buildEndpointUrl(gEndpointBase); let gTemplateName = "openai"; let gCustomTemplate = ""; let gCustomSystemPrompt = DEFAULT_CUSTOM_SYSTEM_PROMPT; @@ -39,6 +42,28 @@ let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); let gCache = new Map(); let gCacheLoaded = false; +function normalizeEndpointBase(endpoint) { + if (typeof endpoint !== "string") { + return ""; + } + let base = endpoint.trim(); + if (!base) { + return ""; + } + base = base.replace(/\/v1\/completions\/?$/i, ""); + return base; +} + +function buildEndpointUrl(endpointBase) { + const base = normalizeEndpointBase(endpointBase); + if (!base) { + return ""; + } + const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; + const needsSlash = withScheme.endsWith("/"); + return `${withScheme}${needsSlash ? "" : "/"}v1/completions`; +} + function sha256HexSync(str) { try { const hasher = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); @@ -158,8 +183,12 @@ function loadTemplateSync(name) { } async function setConfig(config = {}) { - if (config.endpoint) { - gEndpoint = config.endpoint; + if (typeof config.endpoint === "string") { + const base = normalizeEndpointBase(config.endpoint); + if (base) { + gEndpointBase = base; + } + gEndpoint = buildEndpointUrl(gEndpointBase); } if (config.templateName) { gTemplateName = config.templateName; @@ -187,6 +216,10 @@ async function setConfig(config = {}) { } else { gTemplateText = await loadTemplate(gTemplateName); } + if (!gEndpoint) { + gEndpoint = buildEndpointUrl(gEndpointBase); + } + aiLog(`[AiClassifier] Endpoint base set to ${gEndpointBase}`, {debug: true}); aiLog(`[AiClassifier] Endpoint set to ${gEndpoint}`, {debug: true}); aiLog(`[AiClassifier] Template set to ${gTemplateName}`, {debug: true}); } @@ -344,4 +377,4 @@ async function init() { await loadCache(); } -export { classifyText, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, getCacheSize, init }; +export { buildEndpointUrl, normalizeEndpointBase, classifyText, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, getCacheSize, init }; diff --git a/options/options.html b/options/options.html index ddc5ee0..59ebc83 100644 --- a/options/options.html +++ b/options/options.html @@ -73,6 +73,7 @@
    +

    diff --git a/options/options.js b/options/options.js index d881d6a..30fa5f3 100644 --- a/options/options.js +++ b/options/options.js @@ -99,7 +99,21 @@ document.addEventListener('DOMContentLoaded', async () => { markDirty(); await applyTheme(themeSelect.value); }); - document.getElementById('endpoint').value = defaults.endpoint || 'http://127.0.0.1:5000/v1/completions'; + const endpointInput = document.getElementById('endpoint'); + const endpointPreview = document.getElementById('endpoint-preview'); + const fallbackEndpoint = 'http://127.0.0.1:5000'; + const storedEndpoint = defaults.endpoint || fallbackEndpoint; + const endpointBase = AiClassifier.normalizeEndpointBase(storedEndpoint) || storedEndpoint; + endpointInput.value = endpointBase; + + function updateEndpointPreview() { + const resolved = AiClassifier.buildEndpointUrl(endpointInput.value); + endpointPreview.textContent = resolved + ? `Resolved endpoint: ${resolved}` + : 'Resolved endpoint: (invalid)'; + } + endpointInput.addEventListener('input', updateEndpointPreview); + updateEndpointPreview(); const templates = { openai: browser.i18n.getMessage('template.openai'), @@ -806,7 +820,7 @@ document.addEventListener('DOMContentLoaded', async () => { initialized = true; document.getElementById('save').addEventListener('click', async () => { - const endpoint = document.getElementById('endpoint').value; + const endpoint = endpointInput.value.trim(); const templateName = templateSelect.value; const customTemplateText = customTemplate.value; const customSystemPrompt = systemBox.value; From 2178de9a90ba4ea8c846d839f138de183c0186a2 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 21:07:55 -0600 Subject: [PATCH 25/44] Add bash build script --- AGENTS.md | 1 + README.md | 7 +++-- build-xpi.sh | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100755 build-xpi.sh diff --git a/AGENTS.md b/AGENTS.md index e2a1696..23080ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ This file provides guidelines for codex agents contributing to the Sortana proje - `resources/`: Images and other static files. - `prompt_templates/`: Prompt template files for the AI service. - `build-xpi.ps1`: PowerShell script to package the extension. +- `build-xpi.sh`: Bash script to package the extension. ## Coding Style diff --git a/README.md b/README.md index 2f8f204..a58a799 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ with JSON indicating whether the message meets a specified criterion. - **View reasoning** – inspect why rules matched via the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. -- **Packaging script** – `build-xpi.ps1` builds an XPI ready for installation. +- **Packaging scripts** – `build-xpi.ps1` (PowerShell) or `build-xpi.sh` (bash) build an XPI ready for installation. - **Maintenance tab** – view rule counts, cache entries and clear cached results from the options page. ### Cache Storage @@ -65,8 +65,9 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e 1. Ensure PowerShell is available (for Windows) or adapt the script for other environments. 2. The Bulma stylesheet (v1.0.3) is already included as `options/bulma.css`. -3. Run `powershell ./build-xpi.ps1` from the repository root. The script reads - the version from `manifest.json` and creates an XPI in the `release` folder. +3. Run `powershell ./build-xpi.ps1` or `./build-xpi.sh` from the repository root. + The script reads the version from `manifest.json` and creates an XPI in the + `release` folder. 4. Install the generated XPI in Thunderbird via the Add-ons Manager. During development you can also load the directory as a temporary add-on. 5. To regenerate PNG icons from the SVG sources, run `resources/svg2img.ps1`. diff --git a/build-xpi.sh b/build-xpi.sh new file mode 100755 index 0000000..20c6e15 --- /dev/null +++ b/build-xpi.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +release_dir="$script_dir/release" +manifest="$script_dir/manifest.json" + +if [[ ! -f "$manifest" ]]; then + echo "manifest.json not found at $manifest" >&2 + exit 1 +fi + +if ! command -v zip >/dev/null 2>&1; then + echo "zip is required to build the XPI." >&2 + exit 1 +fi + +if command -v jq >/dev/null 2>&1; then + version="$(jq -r '.version // empty' "$manifest")" +else + if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to read manifest.json without jq." >&2 + exit 1 + fi + version="$(python3 - <<'PY' +import json +import sys +with open(sys.argv[1], 'r', encoding='utf-8') as f: + data = json.load(f) +print(data.get('version', '') or '') +PY +"$manifest")" +fi + +if [[ -z "$version" ]]; then + echo "No version found in manifest.json" >&2 + exit 1 +fi + +mkdir -p "$release_dir" + +xpi_name="sortana-$version.xpi" +zip_path="$release_dir/ai-filter-$version.zip" +xpi_path="$release_dir/$xpi_name" + +rm -f "$zip_path" "$xpi_path" + +mapfile -d '' files < <( + find "$script_dir" -type f \ + ! -name '*.sln' \ + ! -name '*.ps1' \ + ! -name '*.sh' \ + ! -path "$release_dir/*" \ + ! -path "$script_dir/.vs/*" \ + ! -path "$script_dir/.git/*" \ + -printf '%P\0' +) + +if [[ ${#files[@]} -eq 0 ]]; then + echo "No files found to package." >&2 + exit 0 +fi + +for rel in "${files[@]}"; do + full="$script_dir/$rel" + size=$(stat -c '%s' "$full") + echo "Zipping: $rel <- $full ($size bytes)" +done + +( + cd "$script_dir" + printf '%s\n' "${files[@]}" | zip -q -9 -@ "$zip_path" +) + +mv -f "$zip_path" "$xpi_path" + +echo "Built XPI at: $xpi_path" From af6702bceb57fa65fda3d47c218d42b40cc83bfe Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 21:42:57 -0600 Subject: [PATCH 26/44] Add prompt reduction badge to debug diff --- options/options.html | 5 ++- options/options.js | 88 +++++++++++++++++++++++++++----------------- 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/options/options.html b/options/options.html index 59ebc83..b118ee0 100644 --- a/options/options.html +++ b/options/options.html @@ -292,7 +292,10 @@
    
                     
                 
    diff --git a/options/options.js b/options/options.js index 30fa5f3..046c674 100644 --- a/options/options.js +++ b/options/options.js @@ -71,6 +71,7 @@ document.addEventListener('DOMContentLoaded', async () => { const payloadDisplay = document.getElementById('payload-display'); const diffDisplay = document.getElementById('diff-display'); const diffContainer = document.getElementById('diff-container'); + const promptReductionLabel = document.getElementById('prompt-reduction'); let lastFullText = defaults.lastFullText || ''; let lastPromptText = defaults.lastPromptText || ''; @@ -79,22 +80,6 @@ document.addEventListener('DOMContentLoaded', async () => { if (lastPayload) { payloadDisplay.textContent = lastPayload; } - if (lastFullText && lastPromptText && diff_match_patch) { - const dmp = new diff_match_patch(); - dmp.Diff_EditCost = 4; - const diffs = dmp.diff_main(lastFullText, lastPromptText); - dmp.diff_cleanupEfficiency(diffs); - const hasDiff = diffs.some(d => d[0] !== 0); - if (hasDiff) { - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); - diffContainer.classList.remove('is-hidden'); - } else { - diffDisplay.innerHTML = ''; - diffContainer.classList.add('is-hidden'); - } - } else { - diffContainer.classList.add('is-hidden'); - } themeSelect.addEventListener('change', async () => { markDirty(); await applyTheme(themeSelect.value); @@ -164,6 +149,51 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReductionToggle = document.getElementById('token-reduction'); tokenReductionToggle.checked = defaults.tokenReduction === true; + function tokenSavingEnabled() { + return htmlToggle.checked + || stripUrlToggle.checked + || altTextToggle.checked + || collapseWhitespaceToggle.checked + || tokenReductionToggle.checked; + } + + function updatePromptReductionLabel(hasDiff) { + if (!promptReductionLabel) return; + if (!hasDiff || !tokenSavingEnabled() || !lastFullText || !lastPromptText) { + promptReductionLabel.classList.add('is-hidden'); + return; + } + const baseLength = lastFullText.length; + const promptLength = lastPromptText.length; + const percentSaved = baseLength > 0 + ? Math.max(0, Math.round((1 - (promptLength / baseLength)) * 100)) + : 0; + promptReductionLabel.textContent = `Prompt Token Reduction: ${percentSaved}%`; + promptReductionLabel.classList.remove('is-hidden'); + } + + function updateDiffDisplay() { + if (lastFullText && lastPromptText && diff_match_patch) { + const dmp = new diff_match_patch(); + dmp.Diff_EditCost = 4; + const diffs = dmp.diff_main(lastFullText, lastPromptText); + dmp.diff_cleanupEfficiency(diffs); + const hasDiff = diffs.some(d => d[0] !== 0); + if (hasDiff) { + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + diffContainer.classList.remove('is-hidden'); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + } + updatePromptReductionLabel(hasDiff); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + updatePromptReductionLabel(false); + } + } + const debugTabToggle = document.getElementById('show-debug-tab'); const debugTabBtn = document.getElementById('debug-tab-button'); function updateDebugTab() { @@ -174,6 +204,14 @@ document.addEventListener('DOMContentLoaded', async () => { debugTabToggle.addEventListener('change', () => { updateDebugTab(); markDirty(); }); updateDebugTab(); + updateDiffDisplay(); + + [htmlToggle, stripUrlToggle, altTextToggle, collapseWhitespaceToggle, tokenReductionToggle].forEach(toggle => { + toggle.addEventListener('change', () => { + updatePromptReductionLabel(!diffContainer.classList.contains('is-hidden')); + }); + }); + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { @@ -770,23 +808,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (latest.lastFullText !== lastFullText || latest.lastPromptText !== lastPromptText) { lastFullText = latest.lastFullText || ''; lastPromptText = latest.lastPromptText || ''; - if (lastFullText && lastPromptText && diff_match_patch) { - const dmp = new diff_match_patch(); - dmp.Diff_EditCost = 4; - const diffs = dmp.diff_main(lastFullText, lastPromptText); - dmp.diff_cleanupEfficiency(diffs); - const hasDiff = diffs.some(d => d[0] !== 0); - if (hasDiff) { - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); - diffContainer.classList.remove('is-hidden'); - } else { - diffDisplay.innerHTML = ''; - diffContainer.classList.add('is-hidden'); - } - } else { - diffDisplay.innerHTML = ''; - diffContainer.classList.add('is-hidden'); - } + updateDiffDisplay(); } } } catch {} From 9269225a0c5da9e13a024677be8375694ac7d93a Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 22:01:20 -0600 Subject: [PATCH 27/44] Add session error log and transient error icon --- README.md | 7 +++-- background.js | 46 +++++++++++++++++++++-------- options/options.html | 27 +++++++++++++++++ options/options.js | 70 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a58a799..87a3565 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,9 @@ with JSON indicating whether the message meets a specified criterion. - **Rule enable/disable** – temporarily turn a rule off without removing it. - **Account & folder filters** – limit rules to specific accounts or folders. - **Context menu** – apply AI rules from the message list or the message display action button. -- **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red until you dismiss the notification. -- **Error notification** – failed classification displays a notification with a button to clear the error and reset the icon. +- **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red briefly before returning to normal. +- **Error notification** – failed classification displays a notification in Thunderbird. +- **Session error log** – the Errors tab (visible only when errors occur) shows errors recorded since the last add-on start. - **View reasoning** – inspect why rules matched via the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. @@ -88,7 +89,7 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the configured rules. -4. If the toolbar icon shows a red X, click the notification's **Dismiss** button to clear the error. +4. If the toolbar icon shows a red X, it will clear after a few seconds. Open the Errors tab in Options to review the latest failures. ### Example Filters diff --git a/background.js b/background.js index fc585ff..e16d5a3 100644 --- a/background.js +++ b/background.js @@ -19,6 +19,7 @@ let queue = Promise.resolve(); let queuedCount = 0; let processing = false; let iconTimer = null; +let errorTimer = null; let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 }; let currentStart = 0; let logGetTiming = true; @@ -33,8 +34,11 @@ let userTheme = 'auto'; let currentTheme = 'light'; let detectSystemTheme; let errorPending = false; +let errorLog = []; let showDebugTab = false; const ERROR_NOTIFICATION_ID = 'sortana-error'; +const ERROR_ICON_TIMEOUT = 4500; +const MAX_ERROR_LOG = 50; function normalizeRules(rules) { return Array.isArray(rules) ? rules.map(r => { @@ -108,11 +112,33 @@ function showTransientIcon(factory, delay = 1500) { async function clearError() { errorPending = false; - await storage.local.set({ errorPending: false }); + clearTimeout(errorTimer); await browser.notifications.clear(ERROR_NOTIFICATION_ID); updateActionIcon(); } +function recordError(context, err) { + const message = err instanceof Error ? err.message : String(err || 'Unknown error'); + const detail = err instanceof Error ? err.stack : ''; + errorLog.unshift({ + time: Date.now(), + context, + message, + detail + }); + if (errorLog.length > MAX_ERROR_LOG) { + errorLog.length = MAX_ERROR_LOG; + } + errorPending = true; + updateActionIcon(); + clearTimeout(errorTimer); + errorTimer = setTimeout(() => { + errorPending = false; + updateActionIcon(); + }, ERROR_ICON_TIMEOUT); + browser.runtime.sendMessage({ type: 'sortana:errorLogUpdated', count: errorLog.length }).catch(() => {}); +} + function refreshMenuIcons() { browser.menus.update('apply-ai-rules-list', { icons: iconPaths('eye') }); browser.menus.update('apply-ai-rules-display', { icons: iconPaths('eye') }); @@ -382,16 +408,14 @@ async function processMessage(id) { const elapsed = Date.now() - currentStart; currentStart = 0; updateTimingStats(elapsed); - await storage.local.set({ classifyStats: timingStats, errorPending: true }); - errorPending = true; + await storage.local.set({ classifyStats: timingStats }); logger.aiLog("failed to apply AI rules", { level: 'error' }, e); - setIcon(ICONS.error()); + recordError("Failed to apply AI rules", e); browser.notifications.create(ERROR_NOTIFICATION_ID, { type: 'basic', iconUrl: browser.runtime.getURL('resources/img/logo.png'), title: 'Sortana Error', - message: 'Failed to apply AI rules', - buttons: [{ title: 'Dismiss' }] + message: 'Failed to apply AI rules' }); } } @@ -451,7 +475,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending", "showDebugTab"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -465,7 +489,6 @@ async function clearCacheForMessages(idsInput) { if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') { maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; } - errorPending = store.errorPending === true; showDebugTab = store.showDebugTab === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -524,10 +547,6 @@ async function clearCacheForMessages(idsInput) { if (changes.showDebugTab) { showDebugTab = changes.showDebugTab.newValue === true; } - if (changes.errorPending) { - errorPending = changes.errorPending.newValue === true; - updateActionIcon(); - } if (changes.theme) { userTheme = changes.theme.newValue || 'auto'; currentTheme = userTheme === 'auto' ? await detectSystemTheme() : userTheme; @@ -720,6 +739,8 @@ async function clearCacheForMessages(idsInput) { } } else if (msg?.type === "sortana:getQueueCount") { return { count: queuedCount + (processing ? 1 : 0) }; + } else if (msg?.type === "sortana:getErrorLog") { + return { errors: errorLog.slice() }; } else if (msg?.type === "sortana:getTiming") { const t = timingStats; const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0; @@ -751,6 +772,7 @@ async function clearCacheForMessages(idsInput) { // Catch any unhandled rejections window.addEventListener("unhandledrejection", ev => { logger.aiLog("Unhandled promise rejection", { level: 'error' }, ev.reason); + recordError("Unhandled promise rejection", ev.reason); }); browser.notifications.onClicked.addListener(id => { diff --git a/options/options.html b/options/options.html index b118ee0..fd8700c 100644 --- a/options/options.html +++ b/options/options.html @@ -51,6 +51,7 @@
  • Settings
  • Rules
  • Maintenance
  • + @@ -285,6 +286,32 @@ + + +
    + +
    +
    +
    + +
    +
    +
    + +
    +
    +

    +
    +
    diff --git a/options/options.js b/options/options.js index 860c944..5e46857 100644 --- a/options/options.js +++ b/options/options.js @@ -10,6 +10,7 @@ document.addEventListener('DOMContentLoaded', async () => { 'templateName', 'customTemplate', 'customSystemPrompt', + 'model', 'aiParams', 'debugLogging', 'htmlToMarkdown', @@ -100,6 +101,88 @@ document.addEventListener('DOMContentLoaded', async () => { endpointInput.addEventListener('input', updateEndpointPreview); updateEndpointPreview(); + const modelSelect = document.getElementById('model-select'); + const refreshModelsBtn = document.getElementById('refresh-models'); + const modelHelp = document.getElementById('model-help'); + const storedModel = typeof defaults.model === 'string' ? defaults.model : ''; + + function setModelHelp(message = '', isError = false) { + if (!modelHelp) return; + modelHelp.textContent = message; + modelHelp.classList.toggle('is-danger', isError); + } + + function populateModelOptions(models = [], selectedModel = '') { + if (!modelSelect) return; + const modelIds = Array.isArray(models) ? models.filter(Boolean) : []; + modelSelect.innerHTML = ''; + + const noneOpt = document.createElement('option'); + noneOpt.value = ''; + noneOpt.textContent = 'None (omit model)'; + modelSelect.appendChild(noneOpt); + + if (selectedModel && !modelIds.includes(selectedModel)) { + const storedOpt = document.createElement('option'); + storedOpt.value = selectedModel; + storedOpt.textContent = `Stored: ${selectedModel}`; + modelSelect.appendChild(storedOpt); + } + + for (const id of modelIds) { + const opt = document.createElement('option'); + opt.value = id; + opt.textContent = id; + modelSelect.appendChild(opt); + } + + const hasSelected = [...modelSelect.options].some(opt => opt.value === selectedModel); + modelSelect.value = hasSelected ? selectedModel : ''; + } + + async function fetchModels(preferredModel = '') { + if (!modelSelect || !refreshModelsBtn) return; + const modelsUrl = AiClassifier.buildModelsUrl(endpointInput.value); + if (!modelsUrl) { + setModelHelp('Set a valid endpoint to load models.', true); + populateModelOptions([], preferredModel || modelSelect.value); + return; + } + + refreshModelsBtn.disabled = true; + setModelHelp('Loading models...'); + + try { + const response = await fetch(modelsUrl, { method: 'GET' }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + const data = await response.json(); + let models = []; + if (Array.isArray(data?.data)) { + models = data.data.map(model => model?.id ?? model?.name ?? model?.model ?? '').filter(Boolean); + } else if (Array.isArray(data?.models)) { + models = data.models.map(model => model?.id ?? model?.name ?? model?.model ?? '').filter(Boolean); + } else if (Array.isArray(data)) { + models = data.map(model => model?.id ?? model?.name ?? model?.model ?? model).filter(Boolean); + } + models = [...new Set(models)]; + populateModelOptions(models, preferredModel || modelSelect.value); + setModelHelp(models.length ? `Loaded ${models.length} model${models.length === 1 ? '' : 's'}.` : 'No models returned.'); + } catch (e) { + logger.aiLog('[options] failed to load models', { level: 'warn' }, e); + setModelHelp('Failed to load models. Check the endpoint and network.', true); + populateModelOptions([], preferredModel || modelSelect.value); + } finally { + refreshModelsBtn.disabled = false; + } + } + + populateModelOptions([], storedModel); + refreshModelsBtn?.addEventListener('click', () => { + fetchModels(modelSelect.value); + }); + const templates = { openai: browser.i18n.getMessage('template.openai'), qwen: browser.i18n.getMessage('template.qwen'), @@ -276,6 +359,7 @@ document.addEventListener('DOMContentLoaded', async () => { await loadErrors(); updateDiffDisplay(); + await fetchModels(storedModel); [htmlToggle, stripUrlToggle, altTextToggle, collapseWhitespaceToggle, tokenReductionToggle].forEach(toggle => { toggle.addEventListener('change', () => { @@ -914,6 +998,7 @@ document.addEventListener('DOMContentLoaded', async () => { document.getElementById('save').addEventListener('click', async () => { const endpoint = endpointInput.value.trim(); + const model = modelSelect?.value || ''; const templateName = templateSelect.value; const customTemplateText = customTemplate.value; const customSystemPrompt = systemBox.value; @@ -979,10 +1064,10 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.set({ endpoint, model, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); await applyTheme(theme); try { - await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ endpoint, model, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); From 1680ad6c3085a11a36dcb138103f1787367d64d1 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Fri, 30 Jan 2026 02:54:19 -0600 Subject: [PATCH 31/44] Add optional OpenAI auth headers --- AGENTS.md | 1 + README.md | 3 +++ background.js | 7 ++++-- manifest.json | 2 +- modules/AiClassifier.js | 28 ++++++++++++++++++++++- options/dataTransfer.js | 3 +++ options/options.html | 26 ++++++++++++++++++++++ options/options.js | 49 ++++++++++++++++++++++++++++++++++++++--- 8 files changed, 112 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f2ea2d9..aece578 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,7 @@ There are currently no automated tests for this project. If you add tests in the Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. The options page can query `/v1/models` from the same base URL to populate the Model dropdown; selecting **None** omits the `model` field from the request payload. +Advanced options allow an optional API key plus `OpenAI-Organization` and `OpenAI-Project` headers; these headers are only sent when values are provided. Responses are expected to include a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in the response text and ignores any surrounding commentary. ## Documentation diff --git a/README.md b/README.md index 957e5e0..5b908ea 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Configurable endpoint** – set the classification service base URL on the options page. - **Model selection** – load available models from the endpoint and choose one (or omit the model field). +- **Optional OpenAI auth headers** – provide an API key plus optional organization/project headers when needed. - **Prompt templates** – choose between OpenAI/ChatML, Qwen, Mistral, Harmony (gpt-oss), or provide your own custom template. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. @@ -82,6 +83,8 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e 1. Open the add-on's options and set the base URL of your classification service (Sortana will append `/v1/completions`). Use the Model dropdown to load `/v1/models` and select a model or choose **None** to omit the `model` field. + Advanced settings include optional API key, organization, and project headers + for OpenAI-hosted endpoints. 2. Use the **Classification Rules** section to add a criterion and optional actions such as tagging, moving, copying, forwarding, replying, deleting or archiving a message when it matches. Drag rules to diff --git a/background.js b/background.js index aef8cbb..827dec8 100644 --- a/background.js +++ b/background.js @@ -484,7 +484,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "model", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -514,10 +514,13 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } - if (changes.endpoint || changes.model || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { + if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { const config = {}; if (changes.endpoint) config.endpoint = changes.endpoint.newValue; if (changes.model) config.model = changes.model.newValue; + if (changes.apiKey) config.apiKey = changes.apiKey.newValue; + if (changes.openaiOrganization) config.openaiOrganization = changes.openaiOrganization.newValue; + if (changes.openaiProject) config.openaiProject = changes.openaiProject.newValue; if (changes.templateName) config.templateName = changes.templateName.newValue; if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; diff --git a/manifest.json b/manifest.json index a18e7cd..81baae5 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.4.0", + "version": "2.4.1", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index cb68382..b4c0907 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -40,6 +40,9 @@ let gTemplateText = ""; let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); let gModel = ""; +let gApiKey = ""; +let gOpenaiOrganization = ""; +let gOpenaiProject = ""; let gCache = new Map(); let gCacheLoaded = false; @@ -223,6 +226,15 @@ async function setConfig(config = {}) { if (typeof config.model === "string") { gModel = config.model.trim(); } + if (typeof config.apiKey === "string") { + gApiKey = config.apiKey.trim(); + } + if (typeof config.openaiOrganization === "string") { + gOpenaiOrganization = config.openaiOrganization.trim(); + } + if (typeof config.openaiProject === "string") { + gOpenaiProject = config.openaiProject.trim(); + } if (typeof config.debugLogging === "boolean") { setDebug(config.debugLogging); } @@ -241,6 +253,20 @@ async function setConfig(config = {}) { aiLog(`[AiClassifier] Template set to ${gTemplateName}`, {debug: true}); } +function buildAuthHeaders() { + const headers = {}; + if (gApiKey) { + headers.Authorization = `Bearer ${gApiKey}`; + } + if (gOpenaiOrganization) { + headers["OpenAI-Organization"] = gOpenaiOrganization; + } + if (gOpenaiProject) { + headers["OpenAI-Project"] = gOpenaiProject; + } + return headers; +} + function buildSystemPrompt() { return SYSTEM_PREFIX + (gCustomSystemPrompt || DEFAULT_CUSTOM_SYSTEM_PROMPT) + SYSTEM_SUFFIX; } @@ -453,7 +479,7 @@ async function classifyText(text, criterion, cacheKey = null) { try { const response = await fetch(gEndpoint, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", ...buildAuthHeaders() }, body: payload, }); diff --git a/options/dataTransfer.js b/options/dataTransfer.js index fdf096f..393b533 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -4,6 +4,9 @@ const KEY_GROUPS = { settings: [ 'endpoint', 'model', + 'apiKey', + 'openaiOrganization', + 'openaiProject', 'templateName', 'customTemplate', 'customSystemPrompt', diff --git a/options/options.html b/options/options.html index 4c1fd79..2a1431e 100644 --- a/options/options.html +++ b/options/options.html @@ -141,6 +141,32 @@
    -

    Built-in formats use native chat messages over Chat Completions. Custom format sends one templated user message.

    +

    OpenAI Chat uses native chat messages. Other formats send one templated user message over Chat Completions.

    - +
    -

    OpenAI Chat uses native chat messages. Other formats send one templated user message over Chat Completions.

    - +
    From 76195f9a92ef76e07264c036659a0366c5323494 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Sun, 6 Sep 2026 23:29:59 -0500 Subject: [PATCH 42/44] Migrate classifier to chat completions --- .codex | 0 AGENTS.md | 5 +-- README.md | 17 ++++---- _locales/en-US/messages.json | 8 ---- ai-filter.sln | 8 ---- background.js | 6 +-- manifest.json | 2 +- modules/AiClassifier.js | 55 ++++++------------------ options/dataTransfer.js | 2 - options/options.html | 17 -------- options/options.js | 83 ++++++++++++++++++++---------------- prompt_templates/harmony.txt | 21 --------- prompt_templates/mistral.txt | 12 ------ prompt_templates/openai.txt | 14 ------ prompt_templates/qwen.txt | 15 ------- 15 files changed, 73 insertions(+), 192 deletions(-) create mode 100644 .codex delete mode 100644 prompt_templates/harmony.txt delete mode 100644 prompt_templates/mistral.txt delete mode 100644 prompt_templates/openai.txt delete mode 100644 prompt_templates/qwen.txt diff --git a/.codex b/.codex new file mode 100644 index 0000000..e69de29 diff --git a/AGENTS.md b/AGENTS.md index 11876f7..1dcb02d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,6 @@ This file provides guidelines for codex agents contributing to the Sortana proje - `options/`: The options page HTML, JavaScript and bundled Bulma CSS (v1.0.3). - `details.html` and `details.js`: View AI reasoning and clear cache for a message. - `resources/`: Images and other static files. -- `prompt_templates/`: Prompt template files for the AI service (openai, qwen, mistral, harmony). - `build-xpi.ps1`: PowerShell script to package the extension. - `build-xpi.sh`: Bash script to package the extension. - `resources/svg2img.ps1`: PowerShell script to regenerate themed PNG icons from SVGs. @@ -35,10 +34,10 @@ There are currently no automated tests for this project. If you add tests in the ## Endpoint Notes -Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. +Sortana targets the OpenAI-compatible `/v1/chat/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/chat/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. The options page can query `/v1/models` from the same base URL to populate the Model dropdown; selecting **None** omits the `model` field from the request payload. Advanced options allow an optional API key plus `OpenAI-Organization` and `OpenAI-Project` headers; these headers are only sent when values are provided. -Responses are expected to include a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in the response text and ignores any surrounding commentary. +Requests use `messages` with system and user roles. Responses are expected to include text in `choices[0].message.content` containing a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in that content and ignores any surrounding commentary. ## Documentation diff --git a/README.md b/README.md index fe67874..a0625d8 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,12 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filter rule. It allows you to classify email messages by sending their contents to a configurable -HTTP endpoint. Sortana uses the `/v1/completions` API; the options page stores a base -URL and appends `/v1/completions` when sending requests. The endpoint should respond -with JSON indicating whether the message meets a specified criterion, including a -short reasoning summary. -Responses are parsed by extracting the last JSON object in the response text and +HTTP endpoint. Sortana uses the OpenAI-compatible `/v1/chat/completions` API; the options page stores a base +URL and appends `/v1/chat/completions` when sending requests. Requests contain a system +message and a user message. The endpoint should respond +with text in `choices[0].message.content` containing JSON indicating whether the +message meets a specified criterion, including a short reasoning summary. +Responses are parsed by extracting the last JSON object in that content and expecting a `match` (or `matched`) boolean plus a `reason` string. ## Features @@ -16,10 +17,10 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Configurable endpoint** – set the classification service base URL on the options page. - **Model selection** – load available models from the endpoint and choose one (or omit the model field). - **Optional OpenAI auth headers** – provide an API key plus optional organization/project headers when needed. -- **Prompt templates** – choose between OpenAI/ChatML, Qwen, Mistral, Harmony (gpt-oss), or provide your own custom template. -- **Custom system prompts** – tailor the instructions sent to the model for more precise results. +- **Custom system prompts** – tailor the instructions sent as the system message for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. + Provider-specific advanced fields are forwarded as configured and may be rejected by strict OpenAI endpoints. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. - **Debug tab** – view the last request payload and a diff between the unaltered message text and the final prompt. @@ -83,7 +84,7 @@ Sortana is implemented entirely with documented MailExtension/WebExtension APIs. ## Usage 1. Open the add-on's options and set the base URL of your classification service - (Sortana will append `/v1/completions`). Use the Model dropdown to load + (Sortana will append `/v1/chat/completions`). Use the Model dropdown to load `/v1/models` and select a model or choose **None** to omit the `model` field. Advanced settings include optional API key, organization, and project headers for OpenAI-hosted endpoints. diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index 017e1a3..472956a 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -4,16 +4,8 @@ "doesntMatch": { "message": "doesn't match" }, "options.title": { "message": "AI Filter Options" }, "options.endpoint": { "message": "Endpoint" }, - "options.template": { "message": "Prompt template" }, - "options.customTemplate": { "message": "Custom template" }, "options.systemInstructions": { "message": "System instructions" }, "options.reset": { "message": "Reset to default" }, - "options.placeholders": { "message": "Placeholders: {{system}}, {{email}}, {{query}}" }, - "template.openai": { "message": "OpenAI / ChatML" }, - "template.qwen": { "message": "Qwen" }, - "template.mistral": { "message": "Mistral" }, - "template.harmony": { "message": "Harmony (gpt-oss)" }, - "template.custom": { "message": "Custom" }, "options.save": { "message": "Save" }, "options.debugLogging": { "message": "Enable debug logging" }, "options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" }, diff --git a/ai-filter.sln b/ai-filter.sln index 7ded341..52e32fe 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -39,14 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "en-US", "en-US", "{8BEA7793 _locales\en-US\messages.json = _locales\en-US\messages.json EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_templates", "{86516D53-50D4-4FE2-9D8A-977A8F5EBDBD}" - ProjectSection(SolutionItems) = preProject - prompt_templates\mistral.txt = prompt_templates\mistral.txt - prompt_templates\openai.txt = prompt_templates\openai.txt - prompt_templates\qwen.txt = prompt_templates\qwen.txt - prompt_templates\harmony.txt = prompt_templates\harmony.txt - EndProjectSection -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{68A87938-5C2B-49F5-8AAA-8A34FBBFD854}" ProjectSection(SolutionItems) = preProject resources\svg2img.ps1 = resources\svg2img.ps1 diff --git a/background.js b/background.js index b2efc89..5cfc112 100644 --- a/background.js +++ b/background.js @@ -484,7 +484,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -514,15 +514,13 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } - if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { + if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { const config = {}; if (changes.endpoint) config.endpoint = changes.endpoint.newValue; if (changes.model) config.model = changes.model.newValue; if (changes.apiKey) config.apiKey = changes.apiKey.newValue; if (changes.openaiOrganization) config.openaiOrganization = changes.openaiOrganization.newValue; if (changes.openaiProject) config.openaiProject = changes.openaiProject.newValue; - if (changes.templateName) config.templateName = changes.templateName.newValue; - if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; if (changes.aiParams) { config.aiParams = changes.aiParams.newValue; diff --git a/manifest.json b/manifest.json index 5dd0e72..139e60e 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.4.3", + "version": "3.0.0", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index dbbde92..583eec0 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -4,7 +4,7 @@ import { DEFAULT_AI_PARAMS } from "./defaultParams.js"; const storage = (globalThis.messenger ?? globalThis.browser).storage; -const COMPLETIONS_PATH = "/v1/completions"; +const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; const MODELS_PATH = "/v1/models"; const SYSTEM_PREFIX = `You are an email-classification assistant. @@ -22,10 +22,7 @@ Do not add any other keys, text, or formatting.`; let gEndpointBase = "http://127.0.0.1:5000"; let gEndpoint = buildEndpointUrl(gEndpointBase); -let gTemplateName = "openai"; -let gCustomTemplate = ""; let gCustomSystemPrompt = DEFAULT_CUSTOM_SYSTEM_PROMPT; -let gTemplateText = ""; let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); let gModel = ""; @@ -44,7 +41,7 @@ function normalizeEndpointBase(endpoint) { if (!base) { return ""; } - base = base.replace(/\/v1\/(completions|models)\/?$/i, ""); + base = base.replace(/\/v1\/(chat\/completions|completions|models)\/?$/i, ""); return base; } @@ -55,7 +52,7 @@ function buildEndpointUrl(endpointBase) { } const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; const needsSlash = withScheme.endsWith("/"); - const path = COMPLETIONS_PATH.replace(/^\//, ""); + const path = CHAT_COMPLETIONS_PATH.replace(/^\//, ""); return `${withScheme}${needsSlash ? "" : "/"}${path}`; } @@ -147,21 +144,6 @@ async function saveCache(updatedKey, updatedValue) { } -async function loadTemplate(name) { - try { - const url = typeof browser !== "undefined" && browser.runtime?.getURL - ? browser.runtime.getURL(`prompt_templates/${name}.txt`) - : `resource://aifilter/prompt_templates/${name}.txt`; - const res = await fetch(url); - if (res.ok) { - return await res.text(); - } - } catch (e) { - aiLog(`Failed to load template '${name}':`, {level: 'error'}, e); - } - return ""; -} - async function setConfig(config = {}) { if (typeof config.endpoint === "string") { const base = normalizeEndpointBase(config.endpoint); @@ -170,12 +152,6 @@ async function setConfig(config = {}) { } gEndpoint = buildEndpointUrl(gEndpointBase); } - if (config.templateName) { - gTemplateName = config.templateName; - } - if (typeof config.customTemplate === "string") { - gCustomTemplate = config.customTemplate; - } if (typeof config.customSystemPrompt === "string") { gCustomSystemPrompt = config.customSystemPrompt; } @@ -201,17 +177,11 @@ async function setConfig(config = {}) { if (typeof config.debugLogging === "boolean") { setDebug(config.debugLogging); } - if (gTemplateName === "custom") { - gTemplateText = gCustomTemplate; - } else { - gTemplateText = await loadTemplate(gTemplateName); - } if (!gEndpoint) { gEndpoint = buildEndpointUrl(gEndpointBase); } aiLog(`[AiClassifier] Endpoint base set to ${gEndpointBase}`, {debug: true}); aiLog(`[AiClassifier] Endpoint set to ${gEndpoint}`, {debug: true}); - aiLog(`[AiClassifier] Template set to ${gTemplateName}`, {debug: true}); } function buildAuthHeaders() { @@ -234,13 +204,7 @@ function buildSystemPrompt() { function buildPrompt(body, criterion) { aiLog(`[AiClassifier] Building prompt with criterion: "${criterion}"`, {debug: true}); - const data = { - system: buildSystemPrompt(), - email: body, - query: criterion, - }; - let template = gTemplateText || ""; - return template.replace(/{{\s*(\w+)\s*}}/g, (m, key) => data[key] || ""); + return `**Email Contents**\n\`\`\`\n${body}\n\`\`\`\nClassification Criterion: ${criterion}`; } function getCachedResult(cacheKey) { @@ -265,7 +229,10 @@ function getReason(cacheKey) { function buildPayload(text, criterion) { let payloadObj = Object.assign({ - prompt: buildPrompt(text, criterion) + messages: [ + { role: "system", content: buildSystemPrompt() }, + { role: "user", content: buildPrompt(text, criterion) } + ] }, gAiParams); if (gModel) { payloadObj.model = gModel; @@ -337,7 +304,11 @@ function extractLastJsonObject(text) { } function parseMatch(result) { - const rawText = result.choices?.[0]?.text || ""; + const rawText = result.choices?.[0]?.message?.content; + if (typeof rawText !== "string") { + reportParseError("Chat response missing text content.", JSON.stringify(result).slice(0, 800)); + return { matched: false, reason: "" }; + } const candidate = extractLastJsonObject(rawText); if (!candidate) { reportParseError("No JSON object found in AI response.", rawText.slice(0, 800)); diff --git a/options/dataTransfer.js b/options/dataTransfer.js index 393b533..2c0b26e 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -7,8 +7,6 @@ const KEY_GROUPS = { 'apiKey', 'openaiOrganization', 'openaiProject', - 'templateName', - 'customTemplate', 'customSystemPrompt', 'aiParams', 'debugLogging', diff --git a/options/options.html b/options/options.html index cfc370d..cd3d581 100644 --- a/options/options.html +++ b/options/options.html @@ -92,23 +92,6 @@

    -
    - -
    -
    - -
    -
    -
    - - -
    diff --git a/options/options.js b/options/options.js index 9a61542..2a8d3c4 100644 --- a/options/options.js +++ b/options/options.js @@ -7,8 +7,6 @@ document.addEventListener('DOMContentLoaded', async () => { const { DEFAULT_AI_PARAMS } = await import(browser.runtime.getURL('modules/defaultParams.js')); const defaults = await storage.local.get([ 'endpoint', - 'templateName', - 'customTemplate', 'customSystemPrompt', 'model', 'apiKey', @@ -163,21 +161,47 @@ document.addEventListener('DOMContentLoaded', async () => { async function fetchModels(preferredModel = '') { if (!modelSelect || !refreshModelsBtn) return; const modelsUrl = AiClassifier.buildModelsUrl(endpointInput.value); + const selectedModel = preferredModel || modelSelect.value; if (!modelsUrl) { + logger.aiLog('[options] model refresh skipped: invalid endpoint', { level: 'warn' }, { + endpoint: endpointInput.value + }); setModelHelp('Set a valid endpoint to load models.', true); - populateModelOptions([], preferredModel || modelSelect.value); + populateModelOptions([], selectedModel); return; } refreshModelsBtn.disabled = true; setModelHelp('Loading models...'); + const headers = buildAuthHeaders(); + logger.aiLog('[options] loading models', {}, { + endpoint: endpointInput.value, + modelsUrl, + selectedModel, + hasAuthorization: typeof headers.Authorization === 'string', + hasOrganization: typeof headers['OpenAI-Organization'] === 'string', + hasProject: typeof headers['OpenAI-Project'] === 'string' + }); try { - const response = await fetch(modelsUrl, { method: 'GET', headers: buildAuthHeaders() }); + const response = await fetch(modelsUrl, { method: 'GET', headers }); + logger.aiLog('[options] model refresh response received', {}, { + status: response.status, + ok: response.ok, + contentType: response.headers.get('content-type') || '' + }); if (!response.ok) { throw new Error(`HTTP ${response.status}`); } const data = await response.json(); + logger.aiLog('[options] model refresh payload parsed', {}, { + payloadType: Array.isArray(data) ? 'array' : typeof data, + hasDataArray: Array.isArray(data?.data), + hasModelsArray: Array.isArray(data?.models), + topLevelKeys: data && typeof data === 'object' && !Array.isArray(data) + ? Object.keys(data) + : [] + }); let models = []; if (Array.isArray(data?.data)) { models = data.data.map(model => model?.id ?? model?.name ?? model?.model ?? '').filter(Boolean); @@ -187,12 +211,23 @@ document.addEventListener('DOMContentLoaded', async () => { models = data.map(model => model?.id ?? model?.name ?? model?.model ?? model).filter(Boolean); } models = [...new Set(models)]; - populateModelOptions(models, preferredModel || modelSelect.value); + logger.aiLog('[options] model refresh parsed models', {}, { + count: models.length, + models + }); + populateModelOptions(models, selectedModel); setModelHelp(models.length ? `Loaded ${models.length} model${models.length === 1 ? '' : 's'}.` : 'No models returned.'); } catch (e) { logger.aiLog('[options] failed to load models', { level: 'warn' }, e); - setModelHelp('Failed to load models. Check the endpoint and network.', true); - populateModelOptions([], preferredModel || modelSelect.value); + const message = String(e?.message || e || ''); + const isLikelyCors = e instanceof TypeError || /Failed to fetch|NetworkError|Load failed/i.test(message); + setModelHelp( + isLikelyCors + ? 'Failed to load models. The server responded, but Thunderbird may have blocked access to the response. Check CORS on the AI server and verify the endpoint is reachable from the add-on.' + : 'Failed to load models. Check the endpoint, HTTP status, and network.', + true + ); + populateModelOptions([], selectedModel); } finally { refreshModelsBtn.disabled = false; } @@ -200,35 +235,10 @@ document.addEventListener('DOMContentLoaded', async () => { populateModelOptions([], storedModel); refreshModelsBtn?.addEventListener('click', () => { + logger.aiLog('[options] model refresh button clicked'); fetchModels(modelSelect.value); }); - const templates = { - openai: browser.i18n.getMessage('template.openai'), - qwen: browser.i18n.getMessage('template.qwen'), - mistral: browser.i18n.getMessage('template.mistral'), - harmony: browser.i18n.getMessage('template.harmony'), - custom: browser.i18n.getMessage('template.custom') - }; - const templateSelect = document.getElementById('template'); - for (const [value, label] of Object.entries(templates)) { - const opt = document.createElement('option'); - opt.value = value; - opt.textContent = label; - templateSelect.appendChild(opt); - } - templateSelect.value = defaults.templateName || 'openai'; - - const customBox = document.getElementById('custom-template-container'); - const customTemplate = document.getElementById('custom-template'); - customTemplate.value = defaults.customTemplate || ''; - - function updateVisibility() { - customBox.classList.toggle('is-hidden', templateSelect.value !== 'custom'); - } - templateSelect.addEventListener('change', updateVisibility); - updateVisibility(); - const advancedBox = document.getElementById('advanced-options'); const advancedBtn = document.getElementById('toggle-advanced'); advancedBtn.addEventListener('click', () => { @@ -1047,8 +1057,6 @@ document.addEventListener('DOMContentLoaded', async () => { const apiKey = apiKeyInput?.value.trim() || ''; const openaiOrganization = openaiOrgInput?.value.trim() || ''; const openaiProject = openaiProjectInput?.value.trim() || ''; - const templateName = templateSelect.value; - const customTemplateText = customTemplate.value; const customSystemPrompt = systemBox.value; const aiParamsSave = {}; for (const key of Object.keys(DEFAULT_AI_PARAMS)) { @@ -1112,10 +1120,11 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, model, apiKey, openaiOrganization, openaiProject, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.set({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.remove(['templateName', 'customTemplate']); await applyTheme(theme); try { - await AiClassifier.setConfig({ endpoint, model, apiKey, openaiOrganization, openaiProject, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); diff --git a/prompt_templates/harmony.txt b/prompt_templates/harmony.txt deleted file mode 100644 index 5054543..0000000 --- a/prompt_templates/harmony.txt +++ /dev/null @@ -1,21 +0,0 @@ -<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. -Knowledge cutoff: 2024-06 -Current date: 2025-06-28 - -Reasoning: medium - -# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|> -<|start|>developer<|message|># Instructions - -{{system}}<|end|> -<|start|>user<|message|>**Email Contents** -``` -{{email}} -``` -Classification Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting.<|end|> -<|start|>assistant diff --git a/prompt_templates/mistral.txt b/prompt_templates/mistral.txt deleted file mode 100644 index 1ee56fa..0000000 --- a/prompt_templates/mistral.txt +++ /dev/null @@ -1,12 +0,0 @@ -[INST] {{system}} - -Email: -{{email}} - -Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting. -[/INST] diff --git a/prompt_templates/openai.txt b/prompt_templates/openai.txt deleted file mode 100644 index c770394..0000000 --- a/prompt_templates/openai.txt +++ /dev/null @@ -1,14 +0,0 @@ -<|im_start|>system -{{system}}<|im_end|> -<|im_start|>user -**Email Contents** -``` -{{email}} -``` -Classification Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting.<|im_end|> -<|im_start|>assistant diff --git a/prompt_templates/qwen.txt b/prompt_templates/qwen.txt deleted file mode 100644 index 1aa0ef0..0000000 --- a/prompt_templates/qwen.txt +++ /dev/null @@ -1,15 +0,0 @@ -<|im_start|>system -{{system}} -<|im_end|> -<|im_start|>user -Email: -{{email}} - -Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting. -<|im_end|> -<|im_start|>assistant From 9a5404cb629df3c5808cdc812044ef91196008c0 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Wed, 7 Oct 2026 23:57:49 -0500 Subject: [PATCH 43/44] Add SystemOne classification support and release version 4.0.0 --- AGENTS.md | 59 ++- README.md | 104 +++++- background.js | 82 ++-- details.html | 27 ++ details.js | 34 +- manifest.json | 2 +- modules/AiClassifier.js | 559 +++++++--------------------- modules/backends/chatCompletions.js | 81 ++++ modules/backends/systemOne.js | 17 + modules/classificationConfig.js | 32 ++ options/dataTransfer.js | 8 +- options/options.html | 21 +- options/options.js | 131 ++++--- tests/cacheRestore.test.mjs | 84 +++++ tests/classification.test.mjs | 167 +++++++++ tests/ruleExecution.test.mjs | 80 ++++ 16 files changed, 916 insertions(+), 572 deletions(-) create mode 100644 modules/backends/chatCompletions.js create mode 100644 modules/backends/systemOne.js create mode 100644 modules/classificationConfig.js create mode 100644 tests/cacheRestore.test.mjs create mode 100644 tests/classification.test.mjs create mode 100644 tests/ruleExecution.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 1dcb02d..673e422 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ This file provides guidelines for codex agents contributing to the Sortana proje - `modules/`: Contains reusable JavaScript modules such as `AiClassifier.js`, `defaultParams.js` and `themeUtils.js`. - `options/`: The options page HTML, JavaScript and bundled Bulma CSS (v1.0.3). -- `details.html` and `details.js`: View AI reasoning and clear cache for a message. +- `details.html` and `details.js`: View classification evidence and clear cache for a message. - `resources/`: Images and other static files. - `build-xpi.ps1`: PowerShell script to package the extension. - `build-xpi.sh`: Bash script to package the extension. @@ -30,14 +30,42 @@ This file provides guidelines for codex agents contributing to the Sortana proje ## Testing -There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. Do **not** run the `ps1` build script or the SVG processing script. +Run the dependency-free suite with Node.js 22 or newer: `node --test tests/*.test.mjs`. +The tests mock extension storage, HTTP responses, and Thunderbird rule execution. +Manual verification in Thunderbird is still required for UI and real-server integration. +Do **not** run the `ps1` build script or the SVG processing script. ## Endpoint Notes -Sortana targets the OpenAI-compatible `/v1/chat/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/chat/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. -The options page can query `/v1/models` from the same base URL to populate the Model dropdown; selecting **None** omits the `model` field from the request payload. -Advanced options allow an optional API key plus `OpenAI-Organization` and `OpenAI-Project` headers; these headers are only sent when values are provided. -Requests use `messages` with system and user roles. Responses are expected to include text in `choices[0].message.content` containing a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in that content and ignores any surrounding commentary. +Sortana supports `apiType: "chat-completions"` (default) and `"systemone"`. +Plain adapter modules under `modules/backends/` own request construction and response +validation; `AiClassifier` owns shared transport, configuration snapshots, threshold +policy, and caching. The stored endpoint is a base URL. Normalization accepts `/v1` +and recognized full API paths, preserves proxy prefixes, and defaults to HTTPS +when no scheme is supplied. Model discovery uses `/v1/models`; **None** omits `model`. +Bearer authentication is optional for both APIs. Organization/project headers are +sent only for Chat Completions. + +Chat Completions sends system/user messages and extracts the last JSON object from +`choices[0].message.content`, requiring a `match` or `matched` boolean. SystemOne +sends prepared email text as `state` and a single `questions.match` of type `noul`. +It requires `answers.match.type === "noul"` and a finite numeric `noul` in `[0, 1]`. +No chat generation parameters or system prompt are sent to SystemOne. + +The global `decisionThreshold` defaults to 0.5; optional rule-level +`decisionThreshold` overrides it. UI percentages are stored as numbers in `[0, 1]`. +Matching uses `probability >= threshold` before display rounding. Details displays +probabilities in a progress bar with the percentage inside; threshold and model +appear in its tooltip. Results use the current threshold, never fabricated reasoning or action +history. No secondary model calls are made. + +`classify()` returns structured evidence; `classifyText()` remains a boolean wrapper. +Failures throw and are never cached. HTTP errors include the request URL and +JSON server error message when available; SystemOne 404 errors suggest checking +the running server build and decision-model support. The rule runner stops that email, reports the +error, and continues the next queued email. Snapshot settings and rules per email. +`inputCharacterLimit` replaces coupling input truncation to output `max_tokens`; +it applies with aggressive reduction and migrates from the old effective limit. ## Documentation @@ -71,10 +99,21 @@ text extracted from all text parts. ### Cache Strategy -`aiCache` persists classification results. Each key is the SHA‑256 hex of -`"|"` and maps to an object with `matched` and `reason` -properties. Any legacy `aiReasonCache` data is merged into `aiCache` the first -time the add-on loads after an update. +`aiCache` persists version 2 inference evidence. Keys are +`v2::`. +Include backend, endpoint, model, effective request parameters and preprocessing; +exclude secrets, actions and thresholds. Recompute SystemOne matches from cached +probability and current thresholds without automatically executing actions. +Entries retain a hashed message identifier for clearing across configurations. +Version 4.0.0 intentionally breaks legacy cache compatibility: legacy caches +(including `aiReasonCache`) are invalidated because their inference +configuration is unknown. Import/export validates versioned evidence; storage +changes refresh the running cache. Maintenance uses `sortana:getCacheSize` to +count validated entries in the background classifier, rather than raw storage keys. +Full restoration requires matching settings and rules as well as cache evidence; +`tests/cacheRestore.test.mjs` exercises export/reinstall/import through the actual +storage and Details handlers, including changed numeric Thunderbird message IDs. Clearing/importing during inference prevents +that in-flight result from repopulating the replaced cache. ### Icon Set Usage diff --git a/README.md b/README.md index a0625d8..b7fb0f5 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,11 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filter rule. It allows you to classify email messages by sending their contents to a configurable -HTTP endpoint. Sortana uses the OpenAI-compatible `/v1/chat/completions` API; the options page stores a base -URL and appends `/v1/chat/completions` when sending requests. Requests contain a system -message and a user message. The endpoint should respond -with text in `choices[0].message.content` containing JSON indicating whether the -message meets a specified criterion, including a short reasoning summary. -Responses are parsed by extracting the last JSON object in that content and -expecting a `match` (or `matched`) boolean plus a `reason` string. +HTTP endpoint. Choose **Chat Completions** for an OpenAI-compatible +`/v1/chat/completions` service or **SystemOne** for decision models served through +`/v1/systemone`. Chat Completions returns a match boolean and textual explanation; +SystemOne returns a match probability, compared against a configurable threshold. +The options page stores a base URL and shows the resolved request URL. ## Features @@ -19,7 +17,7 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Optional OpenAI auth headers** – provide an API key plus optional organization/project headers when needed. - **Custom system prompts** – tailor the instructions sent as the system message for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. -- **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. +- **Advanced parameters (Chat Completions)** – tune generation settings like temperature, top‑p and more from the options page. Provider-specific advanced fields are forwarded as configured and may be rejected by strict OpenAI endpoints. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. @@ -33,7 +31,7 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red briefly before returning to normal. - **Error notification** – failed classification displays a notification in Thunderbird. - **Session error log** – the Errors tab (visible only when errors occur) shows errors recorded since the last add-on start. -- **View reasoning** – inspect why rules matched via the Details popup. +- **View results** – inspect chat explanations or SystemOne probabilities and thresholds in the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. - **Packaging scripts** – `build-xpi.ps1` (PowerShell) or `build-xpi.sh` (bash) build an XPI ready for installation. @@ -42,19 +40,91 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. ### Cache Storage -Classification results are stored under the `aiCache` key in extension storage. -Each entry maps a SHA‑256 hash of `"|"` to an object -containing `matched` and `reason` fields. Older installations with a separate -`aiReasonCache` will be migrated automatically on startup. +Classification results are stored under `aiCache` using version 2 entries. Keys +combine a hashed Message-ID with a fingerprint of the criterion, backend, +endpoint, selected model, effective request settings, and preprocessing settings. +Credentials and thresholds are excluded. SystemOne probabilities are retained at +full precision; changing a threshold reuses the evidence without another request. +Changing inference settings requires a fresh classification. If the model behind +an unchanged server/model alias changes, clear the cache manually. + +**Version 4.0.0 upgrade note:** legacy cache entries and `aiReasonCache` are discarded because +they do not identify the inference configuration. Settings and rules are retained. +Imported legacy caches are also discarded. To restore cached Details after reinstalling, +export and import **Settings, Rules, and Cache** together: lookups require the +original criterion and matching inference settings. Import refreshes the running +classifier immediately; Maintenance counts its validated, loaded entries. A cache-only +import will not recreate rules or settings. Clearing a message's cache removes its +entries across rules and configurations. + +### SystemOne with llama-server + +Use a llama.cpp build supporting `/v1/systemone` and a compatible decision GGUF; +older builds and ordinary chat models cannot serve this API. A 404 with +`File Not Found` from `/v1/systemone` indicates the route is unavailable: check +the running server build and any reverse-proxy routing. Loading a decision GGUF +alone does not add this route to an older server. For example: + +```sh +llama-server -hf ggml-org/Laya-GGUF +``` + +1. Select **SystemOne** in Settings and set the endpoint to `http://127.0.0.1:8080` + (or your server's address). Refresh models and select one, or choose **None** + for a single loaded model. Router deployments require the appropriate model ID. +2. Set the default match probability threshold (initially **50%**). Each rule has + an optional percentage override; leave it blank to inherit the default. +3. Apply rules normally. One `noul` question is sent per evaluated rule. A rule + matches when the unrounded probability is **greater than or equal to** its + effective threshold. For example, 80% matches an 80% threshold. +4. Open Details to see a probability bar with the percentage inside. Hover over + the bar to see the current threshold and model. This is the + result under current settings, not an action history. Changing thresholds does + not automatically rerun actions. Decision models provide no textual explanation; + no second model is called to invent one. Use Chat Completions for explanations. + +Probabilities are model estimates, not guaranteed accuracy. Evaluate thresholds +against representative labeled emails, including long messages and attachments. +SystemOne uses the existing text extraction and attachment summaries; it does not +send attachment binaries or images. Chat prompts and generation settings remain +saved but are inactive for SystemOne. Optional Bearer authentication works with +both APIs; organization/project headers apply only to Chat Completions. + +**Input character limit** applies when aggressive reduction is enabled, independently +of maximum output tokens. Existing installations inherit their previous effective +character limit; the default is 3686. Character counts do not guarantee that input +fits a model's token context or llama-server batch limits. Leave room for question +instructions and configure the server for the chosen model. + +A request or response-parsing failure stops the remaining rules for that email, +records an error, and allows the queue to continue with the next email. Failed +results are never cached. Actions already completed are not rolled back. + +Upstream: [llama-server SystemOne documentation](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#typesafe-compatible-api-endpoints). + +### Verification + +Run the dependency-free tests with Node.js 22 or newer: + +```sh +node --test tests/*.test.mjs +``` + +Tests cover adapters, thresholds, URLs, settings validation, cache migration and +reuse, request failures, per-email snapshots, and queue behavior with mocked +Thunderbird services. Manual acceptance testing in Thunderbird should verify both +API types, model discovery, rule overrides and ordering, import/export, cache +clearing, Details, and stop-after-match against a running server. ## Architecture Overview Sortana is implemented entirely with documented MailExtension/WebExtension APIs. No custom experiment code or Thunderbird-internal fallback paths are required: - `background.js` loads saved settings, manages the classification queue and listens for new messages. -- `modules/AiClassifier.js` implements the classification logic and cache handling. +- `modules/AiClassifier.js` shares transport, threshold policy, and cache handling; + `modules/backends/` contains the Chat Completions and SystemOne adapters. - `options/` contains the HTML and JavaScript for configuring the endpoint and rules. -- `details.html` / `details.js` present cached reasoning for a message. +- `details.html` / `details.js` present cached classification evidence for a message. - `_locales/` holds localized strings used throughout the UI. ### Key Files @@ -65,7 +135,7 @@ Sortana is implemented entirely with documented MailExtension/WebExtension APIs. | `background.js` | Startup tasks and classification queue management. | | `modules/AiClassifier.js` | Core classification logic and cache handling. | | `options/options.html` and `options.js` | Endpoint and rule configuration UI. | -| `details.html` and `details.js` | View stored reasoning for a message. | +| `details.html` and `details.js` | View stored classification evidence for a message. | | `logger.js` | Colorized logging with optional debug mode. | ## Building @@ -84,7 +154,7 @@ Sortana is implemented entirely with documented MailExtension/WebExtension APIs. ## Usage 1. Open the add-on's options and set the base URL of your classification service - (Sortana will append `/v1/chat/completions`). Use the Model dropdown to load + (Sortana appends the path for the selected API type). Use the Model dropdown to load `/v1/models` and select a model or choose **None** to omit the `model` field. Advanced settings include optional API key, organization, and project headers for OpenAI-hosted endpoints. diff --git a/background.js b/background.js index 5cfc112..1b9f554 100644 --- a/background.js +++ b/background.js @@ -28,7 +28,6 @@ let stripUrlParams = false; let altTextImages = false; let collapseWhitespace = false; let tokenReduction = false; -let maxTokens = 4096; let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; @@ -53,6 +52,7 @@ function normalizeRules(rules) { if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); if (r.copyTarget || r.copyTo) actions.push({ type: 'copy', copyTarget: r.copyTarget || r.copyTo }); const rule = { criterion: r.criterion, actions }; + if (r.decisionThreshold !== undefined) rule.decisionThreshold = r.decisionThreshold; if (r.stopProcessing) rule.stopProcessing = true; if (r.unreadOnly) rule.unreadOnly = true; if (typeof r.minAgeDays === 'number') rule.minAgeDays = r.minAgeDays; @@ -318,10 +318,12 @@ async function processMessage(id) { updateActionIcon(); try { const full = await messenger.messages.getFull(id); + const snapshot = AiClassifier.getConfig(); + const rules = structuredClone(aiRules); const originalText = buildEmailText(full, false); let text = buildEmailText(full); - if (tokenReduction && maxTokens > 0) { - const limit = Math.floor(maxTokens * 0.9); + if (tokenReduction) { + const limit = snapshot.inputCharacterLimit; if (text.length > limit) { text = text.slice(0, limit); } @@ -345,7 +347,7 @@ async function processMessage(id) { identityId = null; } - for (const rule of aiRules) { + for (const rule of rules) { if (rule.enabled === false) { continue; } @@ -372,8 +374,8 @@ async function processMessage(id) { } } } - const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion); - const matched = await AiClassifier.classifyText(text, rule.criterion, cacheKey); + const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion, snapshot); + const { matched } = await AiClassifier.classify(text, rule.criterion, { cacheKey, snapshot, threshold: rule.decisionThreshold ?? snapshot.decisionThreshold }); if (matched) { for (const act of (rule.actions || [])) { if (act.type === 'tag' && act.tagKey) { @@ -456,18 +458,8 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(stored); } - const keys = []; - for (const msg of ids) { - const id = msg?.id ?? msg; - for (const rule of aiRules) { - const key = await AiClassifier.buildCacheKey(id, rule.criterion); - keys.push(key); - } - } - if (keys.length) { - await AiClassifier.removeCacheEntries(keys); - showTransientIcon(ICONS.circle); - } + for (const msg of ids) await AiClassifier.clearCacheForMessage(msg?.id ?? msg); + showTransientIcon(ICONS.circle); } (async () => { @@ -484,8 +476,13 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["apiType", "decisionThreshold", "inputCharacterLimit", "endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); + const { migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); + if (store.inputCharacterLimit === undefined) { + store.inputCharacterLimit = migratedInputLimit(store); + await storage.local.set({ inputCharacterLimit: store.inputCharacterLimit }); + } await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; currentTheme = userTheme === 'auto' ? await detectSystemTheme() : userTheme; @@ -495,9 +492,6 @@ async function clearCacheForMessages(idsInput) { altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; tokenReduction = store.tokenReduction === true; - if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') { - maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; - } showDebugTab = store.showDebugTab === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -514,27 +508,9 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } - if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { - const config = {}; - if (changes.endpoint) config.endpoint = changes.endpoint.newValue; - if (changes.model) config.model = changes.model.newValue; - if (changes.apiKey) config.apiKey = changes.apiKey.newValue; - if (changes.openaiOrganization) config.openaiOrganization = changes.openaiOrganization.newValue; - if (changes.openaiProject) config.openaiProject = changes.openaiProject.newValue; - if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; - if (changes.aiParams) { - config.aiParams = changes.aiParams.newValue; - if (changes.aiParams.newValue && typeof changes.aiParams.newValue.max_tokens !== 'undefined') { - maxTokens = parseInt(changes.aiParams.newValue.max_tokens) || maxTokens; - } - } - if (changes.debugLogging) { - config.debugLogging = changes.debugLogging.newValue === true; - logger.setDebug(config.debugLogging); - } - await AiClassifier.setConfig(config); - logger.aiLog("AiClassifier config updated from storage change", { debug: true }, config); - } + const configChanges = Object.fromEntries(Object.entries(changes).map(([key, change]) => [key, change.newValue])); + AiClassifier.setConfig(configChanges); + if (changes.aiCache) AiClassifier.replaceCache(changes.aiCache.newValue); if (changes.htmlToMarkdown) { htmlToMarkdown = changes.htmlToMarkdown.newValue === true; logger.aiLog("htmlToMarkdown updated from storage change", { debug: true }, htmlToMarkdown); @@ -666,10 +642,10 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("sortana:test – criterion", { debug: true }, criterion); try { - logger.aiLog("Calling AiClassifier.classifyText()", { debug: true }); - const result = await AiClassifier.classifyText(text, criterion); + logger.aiLog("Calling AiClassifier.classify()", { debug: true }); + const result = await AiClassifier.classify(text, criterion, { threshold: msg.decisionThreshold ?? AiClassifier.getConfig().decisionThreshold }); logger.aiLog("classify() returned", { debug: true }, result); - return { match: result }; + return { ...result, match: result.matched }; } catch (err) { logger.aiLog("Error in classify()", { level: 'error' }, err); @@ -716,13 +692,11 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(stored); } const results = []; - for (const rule of aiRules) { - const key = await AiClassifier.buildCacheKey(id, rule.criterion); - const matched = AiClassifier.getCachedResult(key); - const reason = AiClassifier.getReason(key); - if (matched !== null || reason) { - results.push({ criterion: rule.criterion, matched, reason }); - } + const snapshot = AiClassifier.getConfig(); + for (const rule of structuredClone(aiRules)) { + const key = await AiClassifier.buildCacheKey(id, rule.criterion, snapshot); + const detail = AiClassifier.getCachedDetails(key, rule.decisionThreshold ?? snapshot.decisionThreshold); + if (detail) results.push({ criterion: rule.criterion, ...detail }); } return { subject, results }; } catch (e) { @@ -757,6 +731,8 @@ async function clearCacheForMessages(idsInput) { timingStats.last = typeof last === 'number' ? last : -1; await storage.local.set({ classifyStats: timingStats }); return { ok: true }; + } else if (msg?.type === "sortana:getCacheSize") { + return { count: await AiClassifier.getCacheSize() }; } else if (msg?.type === "sortana:getQueueCount") { return { count: queuedCount + (processing ? 1 : 0) }; } else if (msg?.type === "sortana:getErrorLog") { diff --git a/details.html b/details.html index d15a3c9..e84bf2e 100644 --- a/details.html +++ b/details.html @@ -4,6 +4,33 @@ AI Details +
    diff --git a/details.js b/details.js index f306e01..a4534d3 100644 --- a/details.js +++ b/details.js @@ -51,15 +51,39 @@ async function loadMessage(id) { article.className = `message ${color} mb-4`; const header = document.createElement('div'); header.className = 'message-header'; - header.innerHTML = `

    ${r.criterion}

    `; + const criterion = document.createElement('p'); + criterion.textContent = r.criterion; + header.appendChild(criterion); const body = document.createElement('div'); body.className = 'message-body'; const status = document.createElement('p'); status.textContent = r.matched ? 'Matched' : 'Did not match'; - const pre = document.createElement('pre'); - pre.textContent = r.reason || ''; - body.appendChild(status); - body.appendChild(pre); + if (r.backend === 'systemone') { + status.textContent += ' under current threshold'; + status.classList.add('mb-2'); + body.appendChild(status); + const percentage = r.probability * 100; + const bar = document.createElement('div'); + bar.className = `probability-bar ${color}`; + bar.setAttribute('role', 'progressbar'); + bar.setAttribute('aria-label', 'Match probability'); + bar.setAttribute('aria-valuemin', '0'); + bar.setAttribute('aria-valuemax', '100'); + bar.setAttribute('aria-valuenow', String(percentage)); + bar.title = `Threshold: ${r.threshold * 100}%${r.model ? ` · Model: ${r.model}` : ''}`; + const fill = document.createElement('div'); + fill.className = 'probability-fill'; + fill.style.width = `${percentage}%`; + const value = document.createElement('span'); + value.className = 'probability-value'; + value.textContent = `${percentage.toFixed(2)}%`; + bar.append(fill, value); + body.appendChild(bar); + } else { + const pre = document.createElement('pre'); + pre.textContent = r.reason || ''; + body.append(status, pre); + } article.appendChild(header); article.appendChild(body); container.appendChild(article); diff --git a/manifest.json b/manifest.json index 139e60e..e9a3088 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "3.0.0", + "version": "4.0.0", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 583eec0..51951e6 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -1,438 +1,151 @@ "use strict"; -import { aiLog, setDebug } from "../logger.js"; -import { DEFAULT_AI_PARAMS } from "./defaultParams.js"; - +import { aiLog, setDebug } from '../logger.js'; +import { CONFIG_DEFAULTS, validateSettings, validateThreshold } from './classificationConfig.js'; +import * as chat from './backends/chatCompletions.js'; +import * as systemone from './backends/systemOne.js'; const storage = (globalThis.messenger ?? globalThis.browser).storage; +const adapters = { 'chat-completions': chat, systemone }; +let config = structuredClone(CONFIG_DEFAULTS); +let cache = new Map(); +let loaded = false; +let loading; +let generation = 0; -const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; -const MODELS_PATH = "/v1/models"; - -const SYSTEM_PREFIX = `You are an email-classification assistant. -Read the email below and the classification criterion provided by the user. -`; - -const DEFAULT_CUSTOM_SYSTEM_PROMPT = "Determine whether the email satisfies the user's criterion."; - -const SYSTEM_SUFFIX = ` -Return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting.`; - -let gEndpointBase = "http://127.0.0.1:5000"; -let gEndpoint = buildEndpointUrl(gEndpointBase); -let gCustomSystemPrompt = DEFAULT_CUSTOM_SYSTEM_PROMPT; - -let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); -let gModel = ""; -let gApiKey = ""; -let gOpenaiOrganization = ""; -let gOpenaiProject = ""; - -let gCache = new Map(); -let gCacheLoaded = false; - -function normalizeEndpointBase(endpoint) { - if (typeof endpoint !== "string") { - return ""; - } - let base = endpoint.trim(); - if (!base) { - return ""; - } - base = base.replace(/\/v1\/(chat\/completions|completions|models)\/?$/i, ""); - return base; +export function normalizeEndpointBase(endpoint) { + if (typeof endpoint !== 'string' || !endpoint.trim()) return ''; + return endpoint.trim().replace(/\/+$/, '').replace(/\/v1(?:\/(?:chat\/completions|completions|models|systemone))?$/i, ''); } - -function buildEndpointUrl(endpointBase) { - const base = normalizeEndpointBase(endpointBase); - if (!base) { - return ""; - } - const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; - const needsSlash = withScheme.endsWith("/"); - const path = CHAT_COMPLETIONS_PATH.replace(/^\//, ""); - return `${withScheme}${needsSlash ? "" : "/"}${path}`; +function urlFor(endpoint, path) { + const base = normalizeEndpointBase(endpoint); + return base ? `${/^https?:\/\//i.test(base) ? base : `https://${base}`}${path}` : ''; } - -function buildModelsUrl(endpointBase) { - const base = normalizeEndpointBase(endpointBase); - if (!base) { - return ""; - } - const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; - const needsSlash = withScheme.endsWith("/"); - const path = MODELS_PATH.replace(/^\//, ""); - return `${withScheme}${needsSlash ? "" : "/"}${path}`; +export function buildEndpointUrl(endpoint, apiType = 'chat-completions') { + if (!adapters[apiType]) throw new Error('Unknown API type.'); + return urlFor(endpoint, adapters[apiType].path); } - -async function sha256Hex(str) { - try { - const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(str)); - return Array.from(new Uint8Array(buf), b => b.toString(16).padStart(2, "0")).join(""); - } catch (e) { - aiLog("sha256Hex failed", { level: "error" }, e); - return ""; +export function buildModelsUrl(endpoint) { return urlFor(endpoint, '/v1/models'); } +export function getConfig() { return structuredClone(config); } +export function setConfig(update = {}) { + validateSettings(update); + for (const key of Object.keys(CONFIG_DEFAULTS)) { + if (update[key] !== undefined) config[key] = structuredClone(update[key]); } + config.aiParams = { ...CONFIG_DEFAULTS.aiParams, ...config.aiParams }; + for (const key of ['model', 'apiKey', 'openaiOrganization', 'openaiProject']) config[key] = String(config[key]).trim(); + setDebug(config.debugLogging); } - -async function resolveHeaderId(id) { - if (typeof id === "number" && typeof messenger?.messages?.get === "function") { +async function hash(value) { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(value)); + return Array.from(new Uint8Array(digest), b => b.toString(16).padStart(2, '0')).join(''); +} +async function messageHash(id) { + const api = globalThis.messenger ?? globalThis.browser; + if (typeof id === 'number' && api.messages?.get) { + const header = await api.messages.get(id); + id = header.headerMessageId || id; + } + return hash(String(id)); +} +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(k => [k, stable(value[k])])); + return value; +} +export async function buildCacheKey(id, criterion, snapshot = getConfig()) { + const fingerprint = { + backend: snapshot.apiType, endpoint: buildEndpointUrl(snapshot.endpoint, snapshot.apiType), + request: adapters[snapshot.apiType].buildRequest('', criterion, snapshot), + organization: snapshot.apiType === 'chat-completions' ? snapshot.openaiOrganization : '', + project: snapshot.apiType === 'chat-completions' ? snapshot.openaiProject : '', + preprocessing: Object.fromEntries(['htmlToMarkdown', 'stripUrlParams', 'altTextImages', 'collapseWhitespace', 'tokenReduction', 'inputCharacterLimit'].map(k => [k, snapshot[k]])) + }; + return `v2:${await messageHash(id)}:${await hash(JSON.stringify(stable(fingerprint)))}`; +} +export function validCacheEntry(key, entry) { + return /^v2:[a-f0-9]{64}:[a-f0-9]{64}$/.test(key) && entry?.version === 2 && + entry.messageHash === key.split(':')[1] && typeof entry.model === 'string' && + (entry.backend === 'systemone' + ? typeof entry.probability === 'number' && Number.isFinite(entry.probability) && entry.probability >= 0 && entry.probability <= 1 && entry.reason === null + : entry.backend === 'chat-completions' && typeof entry.matched === 'boolean' && entry.probability === null && (entry.reason === null || typeof entry.reason === 'string')); +} +export function replaceCache(data = {}) { + cache = new Map(Object.entries(data || {}).filter(([key, entry]) => validCacheEntry(key, entry))); + loaded = true; + generation++; +} +async function saveCache() { await storage.local.set({ aiCache: Object.fromEntries(cache) }); } +export async function init() { + if (loaded) return; + if (!loading) loading = (async () => { + const { aiCache = {}, aiReasonCache } = await storage.local.get(['aiCache', 'aiReasonCache']); + replaceCache(aiCache); + if (Object.keys(aiCache || {}).length !== cache.size) await saveCache(); + if (aiReasonCache !== undefined) await storage.local.remove('aiReasonCache'); + })().finally(() => { loading = null; }); + await loading; +} +function withPolicy(entry, threshold) { + const isDecision = entry.backend === 'systemone'; + if (isDecision) validateThreshold(threshold); + return { matched: isDecision ? entry.probability >= threshold : entry.matched, + reason: entry.reason, probability: entry.probability, threshold: isDecision ? threshold : null, + backend: entry.backend, model: entry.model }; +} +export function getCachedDetails(key, threshold = config.decisionThreshold) { + const entry = cache.get(key); + return entry ? withPolicy(entry, threshold) : null; +} +export function getCachedResult(key, threshold) { return getCachedDetails(key, threshold)?.matched ?? null; } +export function getReason(key) { return cache.get(key)?.reason ?? null; } +export async function classify(text, criterion, { cacheKey = null, snapshot = getConfig(), threshold = snapshot.decisionThreshold } = {}) { + await init(); + if (snapshot.apiType === 'systemone') validateThreshold(threshold); + const cached = getCachedDetails(cacheKey, threshold); + if (cached) return cached; + const epoch = generation; + const adapter = adapters[snapshot.apiType]; + const payload = adapter.buildRequest(text, criterion, snapshot); + try { await storage.local.set({ lastPayload: payload }); } + catch (error) { aiLog('Failed to save last payload', { level: 'warn' }, error); } + const headers = { 'Content-Type': 'application/json' }; + if (snapshot.apiKey) headers.Authorization = `Bearer ${snapshot.apiKey}`; + if (snapshot.apiType === 'chat-completions') { + if (snapshot.openaiOrganization) headers['OpenAI-Organization'] = snapshot.openaiOrganization; + if (snapshot.openaiProject) headers['OpenAI-Project'] = snapshot.openaiProject; + } + const endpoint = buildEndpointUrl(snapshot.endpoint, snapshot.apiType); + const response = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify(payload) }); + if (!response.ok) { + let detail = ''; try { - const hdr = await messenger.messages.get(id); - if (hdr?.headerMessageId) { - return hdr.headerMessageId; - } - } catch (e) { - aiLog(`Failed to resolve headerMessageId for ${id}`, { level: 'warn' }, e); - } + const body = await response.json(); + const message = body?.error?.message; + if (typeof message === 'string') detail = ` Server response: ${message.slice(0, 300)}.`; + } catch { /* Non-JSON error responses still report the URL and status. */ } + const hint = response.status === 404 && snapshot.apiType === 'systemone' + ? ' Verify that the running server build supports /v1/systemone and the selected decision model.' : ''; + throw new Error(`Classification request failed: HTTP ${response.status} at ${endpoint}.${detail}${hint}`); } - return String(id); -} - -async function buildCacheKey(id, criterion) { - const resolvedId = await resolveHeaderId(id); - return sha256Hex(`${resolvedId}|${criterion}`); -} - -async function loadCache() { - if (gCacheLoaded) { - return; - } - aiLog(`[AiClassifier] Loading cache`, {debug: true}); - try { - const { aiCache, aiReasonCache } = await storage.local.get(["aiCache", "aiReasonCache"]); - if (aiCache) { - for (let [k, v] of Object.entries(aiCache)) { - if (v && typeof v === "object") { - gCache.set(k, { matched: v.matched ?? null, reason: v.reason || "" }); - } else { - gCache.set(k, { matched: v, reason: "" }); - } - } - aiLog(`[AiClassifier] Loaded ${gCache.size} cache entries`, {debug: true}); - } else { - aiLog(`[AiClassifier] Cache is empty`, {debug: true}); - } - if (aiReasonCache) { - aiLog(`[AiClassifier] Migrating ${Object.keys(aiReasonCache).length} reason entries`, {debug: true}); - for (let [k, reason] of Object.entries(aiReasonCache)) { - let entry = gCache.get(k) || { matched: null, reason: "" }; - entry.reason = reason; - gCache.set(k, entry); - } - await storage.local.remove("aiReasonCache"); - await storage.local.set({ aiCache: Object.fromEntries(gCache) }); - } - } catch (e) { - aiLog(`Failed to load cache`, {level: 'error'}, e); - } - gCacheLoaded = true; -} - - -async function saveCache(updatedKey, updatedValue) { - if (typeof updatedKey !== "undefined") { - aiLog(`[AiClassifier] ⮡ Persisting entry '${updatedKey}' → ${updatedValue}`, {debug: true}); - } - try { - await storage.local.set({ aiCache: Object.fromEntries(gCache) }); - } catch (e) { - aiLog(`Failed to save cache`, {level: 'error'}, e); - } -} - - -async function setConfig(config = {}) { - if (typeof config.endpoint === "string") { - const base = normalizeEndpointBase(config.endpoint); - if (base) { - gEndpointBase = base; - } - gEndpoint = buildEndpointUrl(gEndpointBase); - } - if (typeof config.customSystemPrompt === "string") { - gCustomSystemPrompt = config.customSystemPrompt; - } - if (config.aiParams && typeof config.aiParams === "object") { - for (let [k, v] of Object.entries(config.aiParams)) { - if (k in gAiParams && typeof v !== "undefined") { - gAiParams[k] = v; - } - } - } - if (typeof config.model === "string") { - gModel = config.model.trim(); - } - if (typeof config.apiKey === "string") { - gApiKey = config.apiKey.trim(); - } - if (typeof config.openaiOrganization === "string") { - gOpenaiOrganization = config.openaiOrganization.trim(); - } - if (typeof config.openaiProject === "string") { - gOpenaiProject = config.openaiProject.trim(); - } - if (typeof config.debugLogging === "boolean") { - setDebug(config.debugLogging); - } - if (!gEndpoint) { - gEndpoint = buildEndpointUrl(gEndpointBase); - } - aiLog(`[AiClassifier] Endpoint base set to ${gEndpointBase}`, {debug: true}); - aiLog(`[AiClassifier] Endpoint set to ${gEndpoint}`, {debug: true}); -} - -function buildAuthHeaders() { - const headers = {}; - if (gApiKey) { - headers.Authorization = `Bearer ${gApiKey}`; - } - if (gOpenaiOrganization) { - headers["OpenAI-Organization"] = gOpenaiOrganization; - } - if (gOpenaiProject) { - headers["OpenAI-Project"] = gOpenaiProject; - } - return headers; -} - -function buildSystemPrompt() { - return SYSTEM_PREFIX + (gCustomSystemPrompt || DEFAULT_CUSTOM_SYSTEM_PROMPT) + SYSTEM_SUFFIX; -} - -function buildPrompt(body, criterion) { - aiLog(`[AiClassifier] Building prompt with criterion: "${criterion}"`, {debug: true}); - return `**Email Contents**\n\`\`\`\n${body}\n\`\`\`\nClassification Criterion: ${criterion}`; -} - -function getCachedResult(cacheKey) { - if (!gCacheLoaded) { - return null; - } - if (cacheKey && gCache.has(cacheKey)) { - aiLog(`[AiClassifier] Cache hit for key: ${cacheKey}`, {debug: true}); - const entry = gCache.get(cacheKey); - return entry?.matched ?? null; - } - return null; -} - -function getReason(cacheKey) { - if (!gCacheLoaded) { - return null; - } - const entry = gCache.get(cacheKey); - return cacheKey && entry ? entry.reason || null : null; -} - -function buildPayload(text, criterion) { - let payloadObj = Object.assign({ - messages: [ - { role: "system", content: buildSystemPrompt() }, - { role: "user", content: buildPrompt(text, criterion) } - ] - }, gAiParams); - if (gModel) { - payloadObj.model = gModel; - } - return JSON.stringify(payloadObj); -} - -function reportParseError(message, detail) { - try { - const runtime = (globalThis.browser ?? globalThis.messenger)?.runtime; - if (!runtime?.sendMessage) { - return; - } - runtime.sendMessage({ - type: "sortana:recordError", - context: "AI response parsing", - message, - detail - }).catch(() => {}); - } catch (e) { - aiLog("Failed to report parse error", { level: "warn" }, e); - } -} - -function extractLastJsonObject(text) { - let last = null; - let start = -1; - let depth = 0; - let inString = false; - let escape = false; - - for (let i = 0; i < text.length; i += 1) { - const ch = text[i]; - if (inString) { - if (escape) { - escape = false; - continue; - } - if (ch === "\\") { - escape = true; - continue; - } - if (ch === "\"") { - inString = false; - } - continue; - } - if (ch === "\"") { - inString = true; - continue; - } - if (ch === "{") { - if (depth === 0) { - start = i; - } - depth += 1; - continue; - } - if (ch === "}" && depth > 0) { - depth -= 1; - if (depth === 0 && start !== -1) { - last = text.slice(start, i + 1); - start = -1; - } - } - } - - return last; -} - -function parseMatch(result) { - const rawText = result.choices?.[0]?.message?.content; - if (typeof rawText !== "string") { - reportParseError("Chat response missing text content.", JSON.stringify(result).slice(0, 800)); - return { matched: false, reason: "" }; - } - const candidate = extractLastJsonObject(rawText); - if (!candidate) { - reportParseError("No JSON object found in AI response.", rawText.slice(0, 800)); - return { matched: false, reason: "" }; - } - - let obj; - try { - obj = JSON.parse(candidate); - } catch (e) { - reportParseError("Failed to parse JSON from AI response.", candidate.slice(0, 800)); - return { matched: false, reason: "" }; - } - - const matchValue = Object.prototype.hasOwnProperty.call(obj, "match") ? obj.match : obj.matched; - const matched = matchValue === true; - if (matchValue !== true && matchValue !== false) { - reportParseError("AI response missing valid match boolean.", candidate.slice(0, 800)); - } - - const reasonValue = obj.reason ?? obj.reasoning ?? obj.explaination; - const reason = typeof reasonValue === "string" ? reasonValue : ""; - - return { matched, reason }; -} - -function cacheEntry(cacheKey, matched, reason) { - if (!cacheKey) { - return; - } - aiLog(`[AiClassifier] Caching entry '${cacheKey}'`, {debug: true}); - const entry = gCache.get(cacheKey) || { matched: null, reason: "" }; - if (typeof matched === "boolean") { - entry.matched = matched; - } - if (typeof reason === "string") { - entry.reason = reason; - } - gCache.set(cacheKey, entry); - saveCache(cacheKey, entry); -} - -async function removeCacheEntries(keys = []) { - if (!Array.isArray(keys)) { - keys = [keys]; - } - if (!gCacheLoaded) { - await loadCache(); - } - let removed = false; - for (let key of keys) { - if (gCache.delete(key)) { - removed = true; - aiLog(`[AiClassifier] Removed cache entry '${key}'`, {debug: true}); - } - } - if (removed) { + const raw = await response.json(); + const entry = { ...adapter.parseResponse(raw), backend: snapshot.apiType, + model: typeof raw.model === 'string' ? raw.model : snapshot.model, version: 2, + messageHash: cacheKey?.split(':')[1] }; + if (cacheKey && epoch === generation) { + cache.set(cacheKey, entry); await saveCache(); } + return withPolicy(entry, threshold); } - -async function clearCache() { - if (!gCacheLoaded) { - await loadCache(); - } - if (gCache.size > 0) { - gCache.clear(); - await saveCache(); - aiLog(`[AiClassifier] Cleared cache`, {debug: true}); - } +export async function classifyText(text, criterion, cacheKey = null) { return (await classify(text, criterion, { cacheKey })).matched; } +export async function removeCacheEntries(keys = []) { + await init(); generation++; + for (const key of Array.isArray(keys) ? keys : [keys]) cache.delete(key); + await saveCache(); } - -async function getCacheSize() { - if (!gCacheLoaded) { - await loadCache(); - } - return gCache.size; +export async function clearCacheForMessage(id) { + await init(); + const identifier = await messageHash(id); + await removeCacheEntries([...cache].filter(([, entry]) => entry.messageHash === identifier).map(([key]) => key)); } - - -async function classifyText(text, criterion, cacheKey = null) { - if (!gCacheLoaded) { - await loadCache(); - } - const cached = getCachedResult(cacheKey); - if (cached !== null) { - return cached; - } - - const payload = buildPayload(text, criterion); - try { - await storage.local.set({ lastPayload: JSON.parse(payload) }); - } catch (e) { - aiLog('failed to save last payload', { level: 'warn' }, e); - } - - aiLog(`[AiClassifier] Sending classification request to ${gEndpoint}`, {debug: true}); - aiLog(`[AiClassifier] Classification request payload:`, { debug: true }, payload); - - try { - const response = await fetch(gEndpoint, { - method: "POST", - headers: { "Content-Type": "application/json", ...buildAuthHeaders() }, - body: payload, - }); - - if (!response.ok) { - aiLog(`HTTP status ${response.status}`, {level: 'warn'}); - return false; - } - - const result = await response.json(); - aiLog(`[AiClassifier] Received response:`, {debug: true}, result); - const parsed = parseMatch(result); - cacheEntry(cacheKey, parsed.matched, parsed.reason); - return parsed.matched; - } catch (e) { - aiLog(`HTTP request failed`, {level: 'error'}, e); - return false; - } -} - -async function init() { - await loadCache(); -} - -export { buildEndpointUrl, buildModelsUrl, normalizeEndpointBase, classifyText, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, getCacheSize, init }; +export async function clearCache() { await init(); generation++; cache.clear(); await saveCache(); } +export async function getCacheSize() { await init(); return cache.size; } diff --git a/modules/backends/chatCompletions.js b/modules/backends/chatCompletions.js new file mode 100644 index 0000000..38d9771 --- /dev/null +++ b/modules/backends/chatCompletions.js @@ -0,0 +1,81 @@ +const SYSTEM_PREFIX = `You are an email-classification assistant. +Read the email below and the classification criterion provided by the user. +`; + +const DEFAULT_CUSTOM_SYSTEM_PROMPT = "Determine whether the email satisfies the user's criterion."; + +const SYSTEM_SUFFIX = ` +Return ONLY a JSON object on a single line of the form: +{"match": true, "reason": ""} - if the email satisfies the criterion +{"match": false, "reason": ""} - otherwise + +Do not add any other keys, text, or formatting.`; + +function extractLastJsonObject(text) { + let last = null; + let start = -1; + let depth = 0; + let inString = false; + let escape = false; + + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (inString) { + if (escape) { + escape = false; + continue; + } + if (ch === "\\") { + escape = true; + continue; + } + if (ch === "\"") { + inString = false; + } + continue; + } + if (ch === "\"") { + inString = true; + continue; + } + if (ch === "{") { + if (depth === 0) { + start = i; + } + depth += 1; + continue; + } + if (ch === "}" && depth > 0) { + depth -= 1; + if (depth === 0 && start !== -1) { + last = text.slice(start, i + 1); + start = -1; + } + } + } + + return last; +} + +export const path = '/v1/chat/completions'; +export function buildRequest(text, criterion, config) { + const payload = { + messages: [ + { role: 'system', content: SYSTEM_PREFIX + (config.customSystemPrompt || DEFAULT_CUSTOM_SYSTEM_PROMPT) + SYSTEM_SUFFIX }, + { role: 'user', content: `**Email Contents**\n\`\`\`\n${text}\n\`\`\`\nClassification Criterion: ${criterion}` } + ], ...config.aiParams + }; + if (config.model) payload.model = config.model; + return payload; +} +export function parseResponse(result) { + const content = result?.choices?.[0]?.message?.content; + if (typeof content !== 'string') throw new Error('Chat response missing text content.'); + const candidate = extractLastJsonObject(content); + if (!candidate) throw new Error('No JSON object found in AI response.'); + const obj = JSON.parse(candidate); + const matched = Object.hasOwn(obj, 'match') ? obj.match : obj.matched; + if (typeof matched !== 'boolean') throw new Error('AI response missing valid match boolean.'); + const reason = obj.reason ?? obj.reasoning ?? obj.explaination; + return { matched, reason: typeof reason === 'string' ? reason : null, probability: null }; +} diff --git a/modules/backends/systemOne.js b/modules/backends/systemOne.js new file mode 100644 index 0000000..0de2472 --- /dev/null +++ b/modules/backends/systemOne.js @@ -0,0 +1,17 @@ +export const path = '/v1/systemone'; +export function buildRequest(text, criterion, config) { + const payload = { + state: text, + questions: { match: { type: 'noul', instructions: `Does this email satisfy the following criterion: ${criterion}` } } + }; + if (config.model) payload.model = config.model; + return payload; +} +export function parseResponse(result) { + const answer = result?.answers?.match; + if (answer?.type !== 'noul' || typeof answer.noul !== 'number' || + !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) { + throw new Error('SystemOne response must contain answers.match with type noul and a probability from 0 to 1.'); + } + return { matched: null, reason: null, probability: answer.noul }; +} diff --git a/modules/classificationConfig.js b/modules/classificationConfig.js new file mode 100644 index 0000000..3abe7cc --- /dev/null +++ b/modules/classificationConfig.js @@ -0,0 +1,32 @@ +import { DEFAULT_AI_PARAMS } from './defaultParams.js'; +export const CONFIG_DEFAULTS = { + apiType: 'chat-completions', endpoint: 'http://127.0.0.1:5000', model: '', + apiKey: '', openaiOrganization: '', openaiProject: '', customSystemPrompt: '', + decisionThreshold: 0.5, inputCharacterLimit: 3686, aiParams: DEFAULT_AI_PARAMS, + htmlToMarkdown: false, stripUrlParams: false, altTextImages: false, + collapseWhitespace: false, tokenReduction: false, debugLogging: false +}; +export function validateThreshold(value) { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error('Match threshold must be between 0% and 100%.'); + } + return value; +} +export function validateSettings(data) { + if (data.apiType !== undefined && !['chat-completions', 'systemone'].includes(data.apiType)) throw new Error('Unknown API type.'); + if (data.decisionThreshold !== undefined) validateThreshold(data.decisionThreshold); + if (data.inputCharacterLimit !== undefined && (!Number.isSafeInteger(data.inputCharacterLimit) || data.inputCharacterLimit < 1)) { + throw new Error('Input character limit must be a positive integer.'); + } + if (data.aiRules !== undefined) { + if (!Array.isArray(data.aiRules)) throw new Error('Rules must be an array.'); + for (const rule of data.aiRules) { + if (rule.decisionThreshold !== undefined) validateThreshold(rule.decisionThreshold); + } + } +} +export function migratedInputLimit(data) { + if (data.inputCharacterLimit !== undefined) return data.inputCharacterLimit; + const value = Math.floor(Number(data.aiParams?.max_tokens ?? 4096) * 0.9); + return Number.isSafeInteger(value) && value > 0 ? value : 3686; +} diff --git a/options/dataTransfer.js b/options/dataTransfer.js index 2c0b26e..ef2b237 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -1,7 +1,10 @@ "use strict"; +import { validateSettings, migratedInputLimit } from '../modules/classificationConfig.js'; +import { validCacheEntry } from '../modules/AiClassifier.js'; const storage = (globalThis.messenger ?? browser).storage; const KEY_GROUPS = { settings: [ + 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'tokenReduction', 'endpoint', 'model', 'apiKey', @@ -36,12 +39,15 @@ export async function exportData(categories) { URL.revokeObjectURL(url); } -export async function importData(file, categories) { +export async function importData(file, categories = Object.keys(KEY_GROUPS)) { const text = await file.text(); const parsed = JSON.parse(text); const data = {}; for (const key of collectKeys(categories)) { if (key in parsed) data[key] = parsed[key]; } + if (categories.includes('settings') && data.inputCharacterLimit === undefined) data.inputCharacterLimit = migratedInputLimit(data); + validateSettings(data); + if (data.aiCache !== undefined) data.aiCache = Object.fromEntries(Object.entries(data.aiCache || {}).filter(([key, entry]) => validCacheEntry(key, entry))); await storage.local.set(data); } diff --git a/options/options.html b/options/options.html index cd3d581..8091df5 100644 --- a/options/options.html +++ b/options/options.html @@ -64,11 +64,25 @@
    + +

    Settings

    +
    + +
    +
    +
    + + +

    Match when probability is at least this value. Rules may override it. Evaluate thresholds on your own email examples.

    +
    @@ -186,7 +200,12 @@
    - + + +

    Applied with aggressive reduction. Characters are not tokens; leave room for instructions within your model's context limit.

    +
    +
    +
    diff --git a/options/options.js b/options/options.js index 2a8d3c4..d1f71a2 100644 --- a/options/options.js +++ b/options/options.js @@ -5,7 +5,9 @@ document.addEventListener('DOMContentLoaded', async () => { const dataTransfer = await import(browser.runtime.getURL('options/dataTransfer.js')); const { detectSystemTheme } = await import(browser.runtime.getURL('modules/themeUtils.js')); const { DEFAULT_AI_PARAMS } = await import(browser.runtime.getURL('modules/defaultParams.js')); + const { validateSettings, migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); const defaults = await storage.local.get([ + 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'endpoint', 'customSystemPrompt', 'model', @@ -86,6 +88,23 @@ document.addEventListener('DOMContentLoaded', async () => { markDirty(); await applyTheme(themeSelect.value); }); + const apiTypeInput = document.getElementById('api-type'); + apiTypeInput.value = defaults.apiType || 'chat-completions'; + const thresholdInput = document.getElementById('decision-threshold'); + thresholdInput.value = (defaults.decisionThreshold ?? 0.5) * 100; + const inputLimit = document.getElementById('input-character-limit'); + inputLimit.value = migratedInputLimit(defaults); + const settingsError = document.getElementById('settings-error'); + function updateApiControls() { + const decision = apiTypeInput.value === 'systemone'; + document.querySelectorAll('.systemone-only').forEach(el => el.classList.toggle('is-hidden', !decision)); + for (const id of ['system-instructions', 'openai-organization', 'openai-project', ...Object.keys(DEFAULT_AI_PARAMS)]) { + document.getElementById(id)?.closest('.field')?.classList.toggle('is-hidden', decision); + } + document.getElementById('reset-system').classList.toggle('is-hidden', decision); + updateEndpointPreview(); + } + apiTypeInput.addEventListener('change', updateApiControls); const endpointInput = document.getElementById('endpoint'); const endpointPreview = document.getElementById('endpoint-preview'); const fallbackEndpoint = 'http://127.0.0.1:5000'; @@ -94,13 +113,13 @@ document.addEventListener('DOMContentLoaded', async () => { endpointInput.value = endpointBase; function updateEndpointPreview() { - const resolved = AiClassifier.buildEndpointUrl(endpointInput.value); + const resolved = AiClassifier.buildEndpointUrl(endpointInput.value, apiTypeInput.value); endpointPreview.textContent = resolved ? `Resolved endpoint: ${resolved}` : 'Resolved endpoint: (invalid)'; } endpointInput.addEventListener('input', updateEndpointPreview); - updateEndpointPreview(); + updateApiControls(); const modelSelect = document.getElementById('model-select'); const refreshModelsBtn = document.getElementById('refresh-models'); @@ -148,11 +167,11 @@ document.addEventListener('DOMContentLoaded', async () => { headers.Authorization = `Bearer ${apiKey}`; } const organization = openaiOrgInput?.value.trim(); - if (organization) { + if (organization && apiTypeInput.value === 'chat-completions') { headers["OpenAI-Organization"] = organization; } const project = openaiProjectInput?.value.trim(); - if (project) { + if (project && apiTypeInput.value === 'chat-completions') { headers["OpenAI-Project"] = project; } return headers; @@ -467,7 +486,7 @@ document.addEventListener('DOMContentLoaded', async () => { const perDayEl = document.getElementById('per-day'); let timingLogged = false; ruleCountEl.textContent = (defaults.aiRules || []).length; - cacheCountEl.textContent = defaults.aiCache ? Object.keys(defaults.aiCache).length : 0; + cacheCountEl.textContent = '…'; function createActionRow(action = {type: 'tag'}) { const row = document.createElement('div'); @@ -861,14 +880,25 @@ document.addEventListener('DOMContentLoaded', async () => { body.appendChild(folderBox); article.appendChild(header); + const thresholdField = document.createElement('label'); + thresholdField.className = 'field systemone-only'; + thresholdField.classList.toggle('is-hidden', apiTypeInput.value !== 'systemone'); + thresholdField.textContent = 'Match probability threshold (%) — blank uses default'; + const override = document.createElement('input'); + override.className = 'input decision-threshold'; + override.type = 'number'; + override.min = '0'; override.max = '100'; override.step = 'any'; + override.value = rule.decisionThreshold === undefined ? '' : rule.decisionThreshold * 100; + thresholdField.appendChild(override); + body.appendChild(thresholdField); article.appendChild(body); rulesContainer.appendChild(article); } } - addRuleBtn.addEventListener('click', () => { - const data = [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { + function readRules() { + return [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { const criterion = ruleEl.querySelector('.criterion').value; const actions = [...ruleEl.querySelectorAll('.action-row')].map(row => { const type = row.querySelector('select').value; @@ -890,8 +920,11 @@ document.addEventListener('DOMContentLoaded', async () => { if (type === 'flag') { return { type, flagged: row.querySelector('.flag-select').value === 'true' }; } - if (type === 'delete' || type === 'archive') { - return { type }; + if (type === 'forward') { + return { type, address: row.querySelector('.forward-input').value.trim() }; + } + if (type === 'reply') { + return { type, replyType: row.querySelector('.reply-select').value }; } return { type }; }); @@ -903,6 +936,8 @@ document.addEventListener('DOMContentLoaded', async () => { const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); const rule = { criterion, actions, enabled }; + const override = ruleEl.querySelector('.decision-threshold').value; + if (override.trim() !== '') rule.decisionThreshold = Number(override) / 100; if (unreadOnly) rule.unreadOnly = true; if (stopProcessing) rule.stopProcessing = true; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; @@ -911,6 +946,9 @@ document.addEventListener('DOMContentLoaded', async () => { if (folders.length) rule.folders = folders; return rule; }); + } + addRuleBtn.addEventListener('click', () => { + const data = readRules(); data.push({ criterion: '', actions: [], unreadOnly: false, stopProcessing: false, enabled: true, accounts: [], folders: [] }); renderRules(data); }); @@ -927,6 +965,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); if (r.copyTarget || r.copyTo) actions.push({ type: 'copy', copyTarget: r.copyTarget || r.copyTo }); const rule = { criterion: r.criterion, actions }; + if (r.decisionThreshold !== undefined) rule.decisionThreshold = r.decisionThreshold; if (r.stopProcessing) rule.stopProcessing = true; if (r.unreadOnly) rule.unreadOnly = true; if (typeof r.minAgeDays === 'number') rule.minAgeDays = r.minAgeDays; @@ -996,8 +1035,8 @@ document.addEventListener('DOMContentLoaded', async () => { } try { - const { aiCache } = await storage.local.get('aiCache'); - cacheCountEl.textContent = aiCache ? Object.keys(aiCache).length : 0; + const { count } = await browser.runtime.sendMessage({ type: 'sortana:getCacheSize' }); + cacheCountEl.textContent = count; } catch { cacheCountEl.textContent = '?'; } @@ -1044,14 +1083,22 @@ document.addEventListener('DOMContentLoaded', async () => { document.getElementById('import-data').addEventListener('click', () => importInput.click()); importInput.addEventListener('change', async () => { if (importInput.files.length) { - await dataTransfer.importData(importInput.files[0], selectedCategories()); - location.reload(); + try { + await dataTransfer.importData(importInput.files[0], selectedCategories()); + location.reload(); + } catch (error) { + window.alert(`Import failed: ${error.message}`); + } } }); initialized = true; document.getElementById('save').addEventListener('click', async () => { + settingsError.textContent = ''; + const apiType = apiTypeInput.value; + const decisionThreshold = thresholdInput.value.trim() === '' ? NaN : Number(thresholdInput.value) / 100; + const inputCharacterLimit = Number(inputLimit.value); const endpoint = endpointInput.value.trim(); const model = modelSelect?.value || ''; const apiKey = apiKeyInput?.value.trim() || ''; @@ -1068,63 +1115,25 @@ document.addEventListener('DOMContentLoaded', async () => { } const debugLogging = debugToggle.checked; const htmlToMarkdown = htmlToggle.checked; - const rules = [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { - const criterion = ruleEl.querySelector('.criterion').value; - const actions = [...ruleEl.querySelectorAll('.action-row')].map(row => { - const type = row.querySelector('select').value; - if (type === 'tag') { - return { type, tagKey: row.querySelector('.tag-select').value }; - } - if (type === 'move') { - return { type, folder: row.querySelector('.folder-select').value }; - } - if (type === 'copy') { - return { type, copyTarget: row.querySelector('.folder-select').value }; - } - if (type === 'junk') { - return { type, junk: row.querySelector('.junk-select').value === 'true' }; - } - if (type === 'read') { - return { type, read: row.querySelector('.read-select').value === 'true' }; - } - if (type === 'flag') { - return { type, flagged: row.querySelector('.flag-select').value === 'true' }; - } - if (type === 'forward') { - return { type, address: row.querySelector('.forward-input').value.trim() }; - } - if (type === 'reply') { - return { type, replyType: row.querySelector('.reply-select').value }; - } - return { type }; - }); - const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; - const unreadOnly = ruleEl.querySelector('.unread-only')?.checked; - const enabled = ruleEl.dataset.enabled !== 'false'; - const minAgeDays = parseFloat(ruleEl.querySelector('.min-age')?.value); - const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); - const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); - const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, enabled }; - if (unreadOnly) rule.unreadOnly = true; - if (stopProcessing) rule.stopProcessing = true; - if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; - if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; - if (accounts.length) rule.accounts = accounts; - if (folders.length) rule.folders = folders; - return rule; - }).filter(r => r.criterion); + const invalidThreshold = [...document.querySelectorAll('.decision-threshold'), thresholdInput, inputLimit].find(el => !el.checkValidity()); + if (invalidThreshold) { + settingsError.textContent = 'Enter thresholds from 0% to 100% and a positive whole-number character limit.'; + return; + } + const rules = readRules().filter(r => r.criterion); const stripUrlParams = stripUrlToggle.checked; const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + try { validateSettings({ apiType, decisionThreshold, inputCharacterLimit, aiRules: rules }); } + catch (error) { settingsError.textContent = error.message; return; } + await storage.local.set({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); await storage.local.remove(['templateName', 'customTemplate']); await applyTheme(theme); try { - await AiClassifier.setConfig({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); diff --git a/tests/cacheRestore.test.mjs b/tests/cacheRestore.test.mjs new file mode 100644 index 0000000..f5ca74d --- /dev/null +++ b/tests/cacheRestore.test.mjs @@ -0,0 +1,84 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import { readFile } from 'node:fs/promises'; +import { CONFIG_DEFAULTS } from '../modules/classificationConfig.js'; + +const persisted = {}; +const listeners = []; +const storage = { + local: { + async get(keys) { + return Object.fromEntries((Array.isArray(keys) ? keys : [keys]) + .filter(key => key in persisted).map(key => [key, structuredClone(persisted[key])])); + }, + async set(values) { + const changes = {}; + for (const [key, value] of Object.entries(values)) { + changes[key] = { oldValue: persisted[key], newValue: structuredClone(value) }; + persisted[key] = structuredClone(value); + } + for (const listener of listeners) await listener(changes, 'local'); + }, + async remove(keys) { + for (const key of Array.isArray(keys) ? keys : [keys]) delete persisted[key]; + } + }, + onChanged: { addListener(listener) { listeners.push(listener); } } +}; +const messenger = { storage, messages: { async get(id) { + // Numeric Thunderbird IDs may change after reinstall; RFC Message-ID does not. + return { subject: 'Invoice', headerMessageId: 'persistent-message@example.com' }; +} } }; +globalThis.messenger = messenger; +const classifier = await import('../modules/AiClassifier.js'); +const transfer = await import('../options/dataTransfer.js'); +const source = await readFile(new URL('../background.js', import.meta.url), 'utf8'); +let onMessage; +const context = vm.createContext({ + storage, messenger, structuredClone, AiClassifier: classifier, aiRules: [], + logger: { aiLog() {} }, logGetTiming: false, + htmlToMarkdown: false, stripUrlParams: false, altTextImages: false, + collapseWhitespace: false, tokenReduction: false, showDebugTab: false, + browser: { runtime: { onMessage: { addListener(listener) { onMessage = listener; } } } } +}); +// Exercise the actual background storage and Details handlers, not a manual cache reload. +const normalizeStart = source.indexOf('function normalizeRules('); +vm.runInContext(source.slice(normalizeStart, source.indexOf('function iconPaths(', normalizeStart)), context); +const changesStart = source.indexOf('storage.onChanged.addListener('); +vm.runInContext(source.slice(changesStart, source.indexOf('\n\n if (browser.theme', changesStart)), context); +const messageStart = source.indexOf('browser.runtime.onMessage.addListener('); +vm.runInContext(source.slice(messageStart, source.indexOf('\n\n // Automatically classify', messageStart)), context); + +test('export/reinstall/import restores Details immediately with changed Thunderbird IDs', async () => { + const settings = { ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 }; + const rules = [{ criterion: 'invoice', decisionThreshold: 0.8, actions: [] }]; + await storage.local.set({ ...settings, aiRules: rules }); + globalThis.fetch = async () => ({ ok: true, json: async () => ({ answers: { match: { type: 'noul', noul: 0.9 } } }) }); + const key = await classifier.buildCacheKey(1, 'invoice'); + await classifier.classify('Invoice attached', 'invoice', { cacheKey: key, threshold: 0.8 }); + let exported; + const originalDocument = globalThis.document; + const createURL = URL.createObjectURL; + const revokeURL = URL.revokeObjectURL; + URL.createObjectURL = blob => { exported = blob; return 'blob:export'; }; + URL.revokeObjectURL = () => {}; + globalThis.document = { body: { appendChild() {} }, createElement() { return { click() {}, remove() {} }; } }; + try { await transfer.exportData(['settings', 'rules', 'cache']); } + finally { globalThis.document = originalDocument; URL.createObjectURL = createURL; URL.revokeObjectURL = revokeURL; } + await classifier.clearCache(); + for (const key of Object.keys(persisted)) delete persisted[key]; + classifier.setConfig(CONFIG_DEFAULTS); + context.aiRules = []; + assert.equal((await onMessage({ type: 'sortana:getDetails', id: 99 })).results.length, 0); + await transfer.importData(exported, ['settings', 'rules', 'cache']); + assert.equal((await onMessage({ type: 'sortana:getCacheSize' })).count, 1); + const details = await onMessage({ type: 'sortana:getDetails', id: 99 }); + assert.equal(details.results.length, 1); + assert.equal(details.results[0].criterion, 'invoice'); + assert.equal(details.results[0].probability, 0.9); + assert.equal(details.results[0].threshold, 0.8); + assert.equal(details.results[0].matched, true); + globalThis.fetch = () => { throw new Error('Restored cache must avoid inference'); }; + assert.equal((await classifier.classify('Invoice attached', 'invoice', { cacheKey: await classifier.buildCacheKey(99, 'invoice') })).matched, true); +}); diff --git a/tests/classification.test.mjs b/tests/classification.test.mjs new file mode 100644 index 0000000..daf6ea6 --- /dev/null +++ b/tests/classification.test.mjs @@ -0,0 +1,167 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as chat from '../modules/backends/chatCompletions.js'; +import * as decision from '../modules/backends/systemOne.js'; +import { CONFIG_DEFAULTS, migratedInputLimit, validateSettings } from '../modules/classificationConfig.js'; + +const data = { aiCache: { legacy: { matched: true, reason: 'old' } }, aiReasonCache: { legacy: 'old' } }; +globalThis.messenger = { storage: { local: { + async get(keys) { return Object.fromEntries((Array.isArray(keys) ? keys : [keys]).map(key => [key, data[key]])); }, + async set(values) { Object.assign(data, structuredClone(values)); }, + async remove(keys) { for (const key of Array.isArray(keys) ? keys : [keys]) delete data[key]; } +} }, messages: { async get(id) { return { headerMessageId: `message-${id}` }; } } }; +const classifier = await import('../modules/AiClassifier.js'); +const transfer = await import('../options/dataTransfer.js'); +const answer = probability => ({ model: 'served-model', answers: { match: { type: 'noul', noul: probability } } }); +let calls = []; +function serve(body, status = 200) { + globalThis.fetch = async (url, request) => { calls.push({ url, ...request }); return { ok: status === 200, status, json: async () => body }; }; +} + +test('adapters preserve chat prompts and isolate SystemOne payloads', () => { + const config = { ...CONFIG_DEFAULTS, model: 'chosen', customSystemPrompt: 'Custom guidance' }; + const payload = chat.buildRequest('email', 'invoice', config); + assert.match(payload.messages[0].content, /Custom guidance/); + assert.match(payload.messages[1].content, /Classification Criterion: invoice/); + assert.equal(payload.max_tokens, 4096); + assert.deepEqual(decision.buildRequest('email', 'invoice', config), { + state: 'email', model: 'chosen', questions: { match: { type: 'noul', instructions: 'Does this email satisfy the following criterion: invoice' } } + }); + assert.equal('model' in decision.buildRequest('email', 'invoice', CONFIG_DEFAULTS), false); + assert.equal('model' in chat.buildRequest('email', 'invoice', CONFIG_DEFAULTS), false); +}); + +test('parsers validate evidence including zero, one, and malformed answers', () => { + for (const probability of [0, 0.5, 1]) assert.equal(decision.parseResponse(answer(probability)).probability, probability); + for (const probability of [null, '0.5', -0.1, 1.1, Infinity, NaN]) assert.throws(() => decision.parseResponse(answer(probability))); + for (const raw of [null, {}, { answers: { other: { type: 'noul', noul: 0.5 } } }, { answers: { match: { type: 'choice', noul: 0.5 } } }]) assert.throws(() => decision.parseResponse(raw)); + assert.deepEqual(chat.parseResponse({ choices: [{ message: { content: 'text {"match":false} then {"matched":true,"reason":"brace } inside"}' } }] }), { matched: true, reason: 'brace } inside', probability: null }); + for (const content of ['', '{broken}', '{"match":"true"}']) assert.throws(() => chat.parseResponse({ choices: [{ message: { content } }] })); + assert.throws(() => chat.parseResponse(null)); +}); + +test('URLs preserve proxy prefixes and normalize API suffixes', () => { + for (const suffix of ['', '/', '/v1', '/v1/', '/v1/models', '/v1/chat/completions/', '/v1/systemone']) { + assert.equal(classifier.buildEndpointUrl(`http://localhost:8080/proxy${suffix}`, 'systemone'), 'http://localhost:8080/proxy/v1/systemone'); + assert.equal(classifier.buildModelsUrl(`http://localhost:8080/proxy${suffix}`), 'http://localhost:8080/proxy/v1/models'); + } + assert.equal(classifier.buildEndpointUrl('example.com'), 'https://example.com/v1/chat/completions'); + assert.equal(classifier.buildEndpointUrl(''), ''); +}); + +test('settings validation and independent character-limit migration', () => { + assert.equal(migratedInputLimit({}), 3686); + assert.equal(migratedInputLimit({ aiParams: { max_tokens: 1000 } }), 900); + assert.equal(migratedInputLimit({ inputCharacterLimit: 500, aiParams: { max_tokens: 1000 } }), 500); + validateSettings({ decisionThreshold: 0, aiRules: [{ decisionThreshold: 1 }] }); + for (const value of [-1, 2, null, '0.5']) assert.throws(() => validateSettings({ aiRules: [{ decisionThreshold: value }] })); + for (const value of [0, 1.2, NaN]) assert.throws(() => validateSettings({ inputCharacterLimit: value })); +}); + +test('cache migration, inference reuse, fingerprints, clearing and import', async () => { + await classifier.init(); + assert.equal(await classifier.getCacheSize(), 0); + assert.equal(data.aiReasonCache, undefined); + assert.deepEqual(data.aiCache, {}); + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://localhost:8080', apiKey: 'secret', openaiOrganization: 'org', openaiProject: 'project' }); + const snapshot = classifier.getConfig(); + const key = await classifier.buildCacheKey(1, 'invoice', snapshot); + serve(answer(0.75)); calls = []; + const first = await classifier.classify('email', 'invoice', { snapshot, cacheKey: key, threshold: 0.75 }); + assert.equal(first.matched, true); + assert.equal(first.model, 'served-model'); + assert.deepEqual(calls[0].headers, { 'Content-Type': 'application/json', Authorization: 'Bearer secret' }); + const second = await classifier.classify('email', 'invoice', { snapshot, cacheKey: key, threshold: 0.8 }); + assert.equal(second.matched, false); + assert.equal(second.probability, 0.75); + assert.equal(calls.length, 1); + classifier.replaceCache(data.aiCache); + assert.equal(classifier.getCachedDetails(key, 0.75).matched, true); + assert.equal(await classifier.buildCacheKey(1, 'invoice', { ...snapshot, decisionThreshold: 0.9, apiKey: 'new secret' }), key); + for (const change of [{ apiType: 'chat-completions' }, { endpoint: 'http://other' }, { model: 'other' }, { tokenReduction: true }, { inputCharacterLimit: 500 }]) { + assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...snapshot, ...change }), key); + } + const chatSnapshot = { ...snapshot, apiType: 'chat-completions' }; + const chatKey = await classifier.buildCacheKey(1, 'invoice', chatSnapshot); + assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...chatSnapshot, customSystemPrompt: 'other' }), chatKey); + assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...chatSnapshot, aiParams: { ...snapshot.aiParams, temperature: 1 } }), chatKey); + serve({ choices: [{ message: { content: '{"match":false,"reason":"not invoice"}' } }] }); + await classifier.classify('email', 'invoice', { cacheKey: chatKey, snapshot: chatSnapshot }); + assert.equal(calls.at(-1).headers['OpenAI-Organization'], 'org'); + assert.equal(await classifier.getCacheSize(), 2); + const exported = structuredClone(data.aiCache); + await classifier.clearCacheForMessage(1); + assert.equal(await classifier.getCacheSize(), 0); + await transfer.importData({ text: async () => JSON.stringify({ aiCache: { ...exported, legacy: true } }) }, ['cache']); + classifier.replaceCache(data.aiCache); // background storage listener consumes imported replacement + assert.equal(await classifier.getCacheSize(), 2); + await assert.rejects(transfer.importData({ text: async () => JSON.stringify({ decisionThreshold: 2 }) }, ['settings'])); + await assert.rejects(transfer.importData({ text: async () => JSON.stringify({ aiRules: [{ decisionThreshold: -1 }] }) }, ['rules'])); + await classifier.clearCache(); +}); + +test('failures remain uncached and snapshots survive configuration changes', async () => { + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone' }); + const snapshot = classifier.getConfig(); + const key = await classifier.buildCacheKey(2, 'invoice', snapshot); + for (const raw of [null, answer('0.9'), {}]) { + serve(raw); + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot })); + assert.equal(classifier.getCachedDetails(key), null); + } + serve({}, 503); + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /503/); + globalThis.fetch = async () => { throw new Error('network'); }; + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /network/); + globalThis.fetch = async () => ({ ok: true, json: async () => { throw new SyntaxError('bad JSON'); } }); + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /bad JSON/); + classifier.setConfig({ apiType: 'chat-completions', endpoint: 'http://changed' }); + serve(answer(1)); calls = []; + const result = await classifier.classify('email', 'invoice', { cacheKey: key, snapshot }); + assert.equal(result.backend, 'systemone'); + assert.equal(calls[0].url, 'http://127.0.0.1:5000/v1/systemone'); + assert.equal('Authorization' in calls[0].headers, false); + await classifier.clearCache(); +}); + +test('clearing or replacing cache during inference prevents stale repopulation', async () => { + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone' }); + const snapshot = classifier.getConfig(); + const key = await classifier.buildCacheKey(3, 'invoice', snapshot); + for (const replace of [() => classifier.clearCache(), () => classifier.replaceCache({})]) { + let release, started; + const ready = new Promise(resolve => { started = resolve; }); + globalThis.fetch = async () => { + started(); + await new Promise(resolve => { release = resolve; }); + return { ok: true, json: async () => answer(0.9) }; + }; + const pending = classifier.classify('email', 'invoice', { cacheKey: key, snapshot }); + await ready; + await replace(); + release(); + assert.equal((await pending).matched, true); + assert.equal(classifier.getCachedDetails(key), null); + } +}); + +test('settings and rule imports retain thresholds and reject invalid caches', async () => { + const settings = { apiType: 'systemone', decisionThreshold: 0.75, inputCharacterLimit: 5000, tokenReduction: true }; + const rules = [{ criterion: 'invoice', decisionThreshold: 0, actions: [{ type: 'flag', flagged: true }] }, { criterion: 'urgent', actions: [] }]; + await transfer.importData({ text: async () => JSON.stringify({ ...settings, aiRules: rules }) }, ['settings', 'rules']); + for (const [key, value] of Object.entries(settings)) assert.equal(data[key], value); + assert.deepEqual(data.aiRules, rules); + const key = await classifier.buildCacheKey(4, 'invoice'); + assert.equal(classifier.validCacheEntry(key, { version: 2, backend: 'systemone', probability: '0.8', reason: null, messageHash: key.split(':')[1], model: '' }), false); +}); + +test('SystemOne 404 reports endpoint, server message, and compatibility guidance', async () => { + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://seinei.wageshouse:5003' }); + serve({ error: { message: 'File Not Found' } }, 404); + await assert.rejects(classifier.classify('synthetic email', 'invoice'), error => { + assert.match(error.message, /HTTP 404 at http:\/\/seinei\.wageshouse:5003\/v1\/systemone/); + assert.match(error.message, /Server response: File Not Found/); + assert.match(error.message, /running server build supports/); + return true; + }); +}); diff --git a/tests/ruleExecution.test.mjs b/tests/ruleExecution.test.mjs new file mode 100644 index 0000000..153a050 --- /dev/null +++ b/tests/ruleExecution.test.mjs @@ -0,0 +1,80 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import { readFile } from 'node:fs/promises'; + +// Run the actual background rule/queue functions with Thunderbird services mocked. +const source = await readFile(new URL('../background.js', import.meta.url), 'utf8'); +const functions = source.slice(source.indexOf('async function processMessage('), source.indexOf('async function clearCacheForMessages(')); +function runner({ fail = false, changeSettings = false } = {}) { + const actions = [], evaluated = [], errors = []; + const rules = [ + { criterion: 'first', decisionThreshold: 0.8, actions: [{ type: 'flag', flagged: true }], stopProcessing: true }, + { criterion: 'second', actions: [{ type: 'read', read: true }] } + ]; + let currentConfig = { apiType: 'systemone', inputCharacterLimit: 5, decisionThreshold: 0.5 }; + const context = vm.createContext({ + structuredClone, Date, Error, + aiRules: rules, queue: Promise.resolve(), queuedCount: 0, processing: false, + currentStart: 0, tokenReduction: true, showDebugTab: false, timingStats: {}, + updateActionIcon() {}, updateTimingStats() {}, showTransientIcon() {}, + buildEmailText() { return 'email contents'; }, ICONS: { circle: 'circle' }, + logger: { aiLog() {} }, recordError(...args) { errors.push(args); }, + ERROR_NOTIFICATION_ID: 'error', + storage: { local: { async set() {} } }, + messenger: { + messages: { + async getFull() { return {}; }, + async get() { + if (changeSettings) { + rules[0].decisionThreshold = 0.1; + currentConfig = { apiType: 'chat-completions', decisionThreshold: 0.1 }; + } + return { tags: [], folder: { accountId: 'account' } }; + }, + async update(id, action) { actions.push({ id, ...action }); } + }, identities: { async list() { return []; } } + }, + browser: { runtime: { getURL: x => x }, notifications: { async create() {} } }, + AiClassifier: { + getConfig() { return structuredClone(currentConfig); }, + async buildCacheKey(id, criterion) { return `${id}:${criterion}`; }, + async classify(text, criterion, options) { + evaluated.push({ text, criterion, ...options }); + if (fail && options.cacheKey.startsWith('1:')) throw new Error('inference failed'); + return { matched: true }; + } + } + }); + vm.runInContext(functions, context); + return { context, actions, evaluated, errors }; +} + +test('failure stops remaining rules for one email, next queued email still acts', async () => { + const { context, actions, evaluated, errors } = runner({ fail: true }); + await vm.runInContext('applyAiRules([1, 2])', context); + assert.deepEqual(evaluated.map(x => x.cacheKey), ['1:first', '2:first']); + assert.deepEqual(actions, [{ id: 2, flagged: true }]); + assert.equal(errors.length, 1); + assert.equal(context.processing, false); + assert.equal(context.queuedCount, 0); +}); + +test('settings and rules remain fixed during an email; reduction uses characters', async () => { + const { context, evaluated, actions } = runner({ changeSettings: true }); + await vm.runInContext('applyAiRules([1])', context); + assert.equal(evaluated[0].text, 'email'); + assert.equal(evaluated[0].threshold, 0.8); + assert.equal(evaluated[0].snapshot.apiType, 'systemone'); + assert.equal(evaluated.length, 1); // stop after first match + assert.equal(actions.length, 1); +}); + +test('inherited thresholds and eligibility preserve ordered rule processing', async () => { + const { context, evaluated, actions } = runner(); + context.aiRules[0].enabled = false; + await vm.runInContext('applyAiRules([1])', context); + assert.equal(evaluated[0].criterion, 'second'); + assert.equal(evaluated[0].threshold, 0.5); + assert.deepEqual(actions, [{ id: 1, read: true }]); +}); From 7edae937b3fe7fcfd2be4ad47c48330469870ec8 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Thu, 8 Oct 2026 09:42:26 -0500 Subject: [PATCH 44/44] Add configurable SystemOne batching and release version 4.1.0 --- AGENTS.md | 27 +++- README.md | 20 ++- background.js | 9 +- manifest.json | 2 +- modules/AiClassifier.js | 76 +++++++++-- modules/backends/systemOne.js | 19 ++- modules/classificationConfig.js | 7 + options/dataTransfer.js | 1 + options/options.html | 9 ++ options/options.js | 27 +++- tests/batching.test.mjs | 219 ++++++++++++++++++++++++++++++++ tests/cacheRestore.test.mjs | 4 +- tests/ruleExecution.test.mjs | 11 +- 13 files changed, 400 insertions(+), 31 deletions(-) create mode 100644 tests/batching.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 673e422..6a3d1e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,8 +48,10 @@ sent only for Chat Completions. Chat Completions sends system/user messages and extracts the last JSON object from `choices[0].message.content`, requiring a `match` or `matched` boolean. SystemOne -sends prepared email text as `state` and a single `questions.match` of type `noul`. -It requires `answers.match.type === "noul"` and a finite numeric `noul` in `[0, 1]`. +sends prepared email text as `state` with `noul` questions identified by stable +`rule_` IDs. The single-question API still uses `match` for backward +compatibility and cache fingerprinting. Every requested answer must have type +`noul` and a finite numeric probability in `[0, 1]`; response order is irrelevant. No chat generation parameters or system prompt are sent to SystemOne. The global `decisionThreshold` defaults to 0.5; optional rule-level @@ -67,6 +69,27 @@ error, and continues the next queued email. Snapshot settings and rules per emai `inputCharacterLimit` replaces coupling input truncation to output `max_tokens`; it applies with aggressive reduction and migrates from the old effective limit. +### SystemOne batching + +`systemOneProcessAll` defaults to true; `systemOneBatchSize` defaults to 8 and must +be a positive safe integer. Only SystemOne shows these controls. The numeric input +is disabled while Process all is enabled, retaining its value. Include both in +settings import/export, startup loading, live updates, and per-email snapshots. + +`classifyRules()` is a lazy ordered async iterator over precomputed candidate keys. +Chat requests stay sequential. SystemOne batches the next N unique uncached keys +(or all), sends state once, validates the whole response, and persists one batch +of evidence before yielding ordered results. Per-rule thresholds remain independent. +Do not change existing cache keys or include batch settings in their fingerprint. +Prepare eligibility and keys before actions, using the existing initial mail metadata. + +Stop-after-match ends action iteration and prevents further requests; already +computed later answers remain cached. A malformed answer fails its entire batch: +no partial cache writes or new actions. Keep cache-generation guards for imports +and clearing during inference. No automatic splitting or retries. Tests in +`tests/batching.test.mjs` cover batching, cache reuse, failure atomicity, and actual +rule-runner stop behavior with the real classifier. + ## Documentation Additional documentation exists outside this repository. diff --git a/README.md b/README.md index b7fb0f5..999d826 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ llama-server -hf ggml-org/Laya-GGUF for a single loaded model. Router deployments require the appropriate model ID. 2. Set the default match probability threshold (initially **50%**). Each rule has an optional percentage override; leave it blank to inherit the default. -3. Apply rules normally. One `noul` question is sent per evaluated rule. A rule +3. Apply rules normally. Each uncached criterion becomes a `noul` question. A rule matches when the unrounded probability is **greater than or equal to** its effective threshold. For example, 80% matches an 80% threshold. 4. Open Details to see a probability bar with the percentage inside. Hover over @@ -83,6 +83,24 @@ llama-server -hf ggml-org/Laya-GGUF not automatically rerun actions. Decision models provide no textual explanation; no second model is called to invent one. Use Chat Completions for explanations. +**SystemOne batching:** **Process all** is enabled by default and sends all eligible, +uncached criteria for an email in one request, with the email state included once. +Turn it off to set **Rules per request** (initially 8); use 1 for sequential +requests. The number is retained while disabled. These controls apply only to +SystemOne; Chat Completions continues processing one rule per request. + +Cached rules consume no batch capacity, and identical criteria share a question +while retaining their individual thresholds. Actions execute in rule order and +honor stop-after-match. A batch can evaluate later rules whose actions are then +skipped; their probabilities remain available in Details. No further batches are +requested after stopping. Changing batch settings preserves existing cached results. + +If any requested answer is missing or invalid, the entire batch fails without +caching its new answers or executing their actions. Previously completed actions +remain applied. Sortana does not automatically retry or split rejected batches; +if the server rejects a large request, disable Process all and reduce Rules per +request. Limits and performance depend on the server and model. + Probabilities are model estimates, not guaranteed accuracy. Evaluate thresholds against representative labeled emails, including long messages and attachments. SystemOne uses the existing text extraction and attachment summaries; it does not diff --git a/background.js b/background.js index 1b9f554..602fe1f 100644 --- a/background.js +++ b/background.js @@ -347,7 +347,8 @@ async function processMessage(id) { identityId = null; } - for (const rule of rules) { + const candidates = []; + for (const [index, rule] of rules.entries()) { if (rule.enabled === false) { continue; } @@ -375,7 +376,9 @@ async function processMessage(id) { } } const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion, snapshot); - const { matched } = await AiClassifier.classify(text, rule.criterion, { cacheKey, snapshot, threshold: rule.decisionThreshold ?? snapshot.decisionThreshold }); + candidates.push({ id: `rule_${index}`, rule, cacheKey }); + } + for await (const { rule, result: { matched } } of AiClassifier.classifyRules(text, candidates, snapshot)) { if (matched) { for (const act of (rule.actions || [])) { if (act.type === 'tag' && act.tagKey) { @@ -476,7 +479,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["apiType", "decisionThreshold", "inputCharacterLimit", "endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["systemOneProcessAll", "systemOneBatchSize", "apiType", "decisionThreshold", "inputCharacterLimit", "endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); const { migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); if (store.inputCharacterLimit === undefined) { diff --git a/manifest.json b/manifest.json index e9a3088..b304bc0 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "4.0.0", + "version": "4.1.0", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 51951e6..603816f 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -97,14 +97,7 @@ export function getCachedDetails(key, threshold = config.decisionThreshold) { } export function getCachedResult(key, threshold) { return getCachedDetails(key, threshold)?.matched ?? null; } export function getReason(key) { return cache.get(key)?.reason ?? null; } -export async function classify(text, criterion, { cacheKey = null, snapshot = getConfig(), threshold = snapshot.decisionThreshold } = {}) { - await init(); - if (snapshot.apiType === 'systemone') validateThreshold(threshold); - const cached = getCachedDetails(cacheKey, threshold); - if (cached) return cached; - const epoch = generation; - const adapter = adapters[snapshot.apiType]; - const payload = adapter.buildRequest(text, criterion, snapshot); +async function sendRequest(payload, snapshot) { try { await storage.local.set({ lastPayload: payload }); } catch (error) { aiLog('Failed to save last payload', { level: 'warn' }, error); } const headers = { 'Content-Type': 'application/json' }; @@ -126,7 +119,18 @@ export async function classify(text, criterion, { cacheKey = null, snapshot = ge ? ' Verify that the running server build supports /v1/systemone and the selected decision model.' : ''; throw new Error(`Classification request failed: HTTP ${response.status} at ${endpoint}.${detail}${hint}`); } - const raw = await response.json(); + return response.json(); +} + +export async function classify(text, criterion, { cacheKey = null, snapshot = getConfig(), threshold = snapshot.decisionThreshold } = {}) { + await init(); + if (snapshot.apiType === 'systemone') validateThreshold(threshold); + const cached = getCachedDetails(cacheKey, threshold); + if (cached) return cached; + const epoch = generation; + const adapter = adapters[snapshot.apiType]; + const payload = adapter.buildRequest(text, criterion, snapshot); + const raw = await sendRequest(payload, snapshot); const entry = { ...adapter.parseResponse(raw), backend: snapshot.apiType, model: typeof raw.model === 'string' ? raw.model : snapshot.model, version: 2, messageHash: cacheKey?.split(':')[1] }; @@ -136,6 +140,60 @@ export async function classify(text, criterion, { cacheKey = null, snapshot = ge } return withPolicy(entry, threshold); } +// Yield lazily so the action runner can stop without starting the next request. +// Candidates have precomputed keys, allowing actions to move/delete the message. +export async function* classifyRules(text, candidates, snapshot = getConfig()) { + await init(); + validateSettings(snapshot); + if (snapshot.apiType !== 'systemone') { + for (const candidate of candidates) { + const result = await classify(text, candidate.rule.criterion, { + cacheKey: candidate.cacheKey, snapshot, + threshold: candidate.rule.decisionThreshold ?? snapshot.decisionThreshold + }); + yield { rule: candidate.rule, result }; + } + return; + } + const limit = snapshot.systemOneProcessAll ? Infinity : snapshot.systemOneBatchSize; + // Keep received evidence locally even if persistence is cleared during this run. + const evidence = new Map(); + for (const candidate of candidates) { + validateThreshold(candidate.rule.decisionThreshold ?? snapshot.decisionThreshold); + const entry = cache.get(candidate.cacheKey); + if (entry) evidence.set(candidate.cacheKey, entry); + } + for (let index = 0; index < candidates.length; index++) { + const candidate = candidates[index]; + if (!evidence.has(candidate.cacheKey)) { + const pending = new Map(); + for (let next = index; next < candidates.length && pending.size < limit; next++) { + const item = candidates[next]; + if (!evidence.has(item.cacheKey) && !pending.has(item.cacheKey)) { + pending.set(item.cacheKey, { ...item, criterion: item.rule.criterion }); + } + } + const questions = [...pending.values()]; + const epoch = generation; + const raw = await sendRequest(systemone.buildBatchRequest(text, questions, snapshot), snapshot); + // Parse the complete batch before accepting any of its evidence. + const parsed = systemone.parseBatchResponse(raw, questions); + const entries = questions.map((question, i) => [question.cacheKey, { + ...parsed[i], backend: snapshot.apiType, version: 2, + model: typeof raw.model === 'string' ? raw.model : snapshot.model, + messageHash: question.cacheKey.split(':')[1] + }]); + for (const [key, entry] of entries) evidence.set(key, entry); + if (epoch === generation) { + for (const [key, entry] of entries) cache.set(key, entry); + await saveCache(); + } + } + yield { rule: candidate.rule, result: withPolicy(evidence.get(candidate.cacheKey), + candidate.rule.decisionThreshold ?? snapshot.decisionThreshold) }; + } +} + export async function classifyText(text, criterion, cacheKey = null) { return (await classify(text, criterion, { cacheKey })).matched; } export async function removeCacheEntries(keys = []) { await init(); generation++; diff --git a/modules/backends/systemOne.js b/modules/backends/systemOne.js index 0de2472..70627f1 100644 --- a/modules/backends/systemOne.js +++ b/modules/backends/systemOne.js @@ -1,17 +1,22 @@ export const path = '/v1/systemone'; export function buildRequest(text, criterion, config) { - const payload = { - state: text, - questions: { match: { type: 'noul', instructions: `Does this email satisfy the following criterion: ${criterion}` } } - }; + return buildBatchRequest(text, [{ id: 'match', criterion }], config); +} +export function buildBatchRequest(text, questions, config) { + const payload = { state: text, questions: Object.fromEntries(questions.map(({ id, criterion }) => + [id, { type: 'noul', instructions: `Does this email satisfy the following criterion: ${criterion}` }])) }; if (config.model) payload.model = config.model; return payload; } -export function parseResponse(result) { - const answer = result?.answers?.match; +export function parseResponse(result) { return parseAnswer(result, 'match'); } +export function parseBatchResponse(result, questions) { + return questions.map(({ id }) => parseAnswer(result, id)); +} +function parseAnswer(result, id) { + const answer = result?.answers?.[id]; if (answer?.type !== 'noul' || typeof answer.noul !== 'number' || !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) { - throw new Error('SystemOne response must contain answers.match with type noul and a probability from 0 to 1.'); + throw new Error(`SystemOne response must contain answers.${id} with type noul and a probability from 0 to 1.`); } return { matched: null, reason: null, probability: answer.noul }; } diff --git a/modules/classificationConfig.js b/modules/classificationConfig.js index 3abe7cc..d6dbb5a 100644 --- a/modules/classificationConfig.js +++ b/modules/classificationConfig.js @@ -2,6 +2,7 @@ import { DEFAULT_AI_PARAMS } from './defaultParams.js'; export const CONFIG_DEFAULTS = { apiType: 'chat-completions', endpoint: 'http://127.0.0.1:5000', model: '', apiKey: '', openaiOrganization: '', openaiProject: '', customSystemPrompt: '', + systemOneProcessAll: true, systemOneBatchSize: 8, decisionThreshold: 0.5, inputCharacterLimit: 3686, aiParams: DEFAULT_AI_PARAMS, htmlToMarkdown: false, stripUrlParams: false, altTextImages: false, collapseWhitespace: false, tokenReduction: false, debugLogging: false @@ -13,6 +14,12 @@ export function validateThreshold(value) { return value; } export function validateSettings(data) { + if (data.systemOneProcessAll !== undefined && typeof data.systemOneProcessAll !== 'boolean') { + throw new Error('Process all must be a boolean.'); + } + if (data.systemOneBatchSize !== undefined && (!Number.isSafeInteger(data.systemOneBatchSize) || data.systemOneBatchSize < 1)) { + throw new Error('Rules per request must be a positive whole number.'); + } if (data.apiType !== undefined && !['chat-completions', 'systemone'].includes(data.apiType)) throw new Error('Unknown API type.'); if (data.decisionThreshold !== undefined) validateThreshold(data.decisionThreshold); if (data.inputCharacterLimit !== undefined && (!Number.isSafeInteger(data.inputCharacterLimit) || data.inputCharacterLimit < 1)) { diff --git a/options/dataTransfer.js b/options/dataTransfer.js index ef2b237..298974c 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -4,6 +4,7 @@ import { validCacheEntry } from '../modules/AiClassifier.js'; const storage = (globalThis.messenger ?? browser).storage; const KEY_GROUPS = { settings: [ + 'systemOneProcessAll', 'systemOneBatchSize', 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'tokenReduction', 'endpoint', 'model', diff --git a/options/options.html b/options/options.html index 8091df5..2ab0c8a 100644 --- a/options/options.html +++ b/options/options.html @@ -78,6 +78,15 @@
    +
    + +

    Group all eligible, uncached rules for each email into one request. Actions still follow rule order and stop-after-match.

    +
    +
    + + +

    Turn off Process all to limit each request. Use 1 for one rule at a time.

    +
    diff --git a/options/options.js b/options/options.js index d1f71a2..124d163 100644 --- a/options/options.js +++ b/options/options.js @@ -7,6 +7,7 @@ document.addEventListener('DOMContentLoaded', async () => { const { DEFAULT_AI_PARAMS } = await import(browser.runtime.getURL('modules/defaultParams.js')); const { validateSettings, migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); const defaults = await storage.local.get([ + 'systemOneProcessAll', 'systemOneBatchSize', 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'endpoint', 'customSystemPrompt', @@ -90,6 +91,15 @@ document.addEventListener('DOMContentLoaded', async () => { }); const apiTypeInput = document.getElementById('api-type'); apiTypeInput.value = defaults.apiType || 'chat-completions'; + const processAllInput = document.getElementById('systemone-process-all'); + const batchSizeInput = document.getElementById('systemone-batch-size'); + processAllInput.checked = defaults.systemOneProcessAll ?? true; + let savedBatchSize = defaults.systemOneBatchSize ?? 8; + batchSizeInput.value = savedBatchSize; + function updateBatchControls() { + batchSizeInput.disabled = processAllInput.checked || apiTypeInput.value !== 'systemone'; + } + processAllInput.addEventListener('change', updateBatchControls); const thresholdInput = document.getElementById('decision-threshold'); thresholdInput.value = (defaults.decisionThreshold ?? 0.5) * 100; const inputLimit = document.getElementById('input-character-limit'); @@ -97,6 +107,7 @@ document.addEventListener('DOMContentLoaded', async () => { const settingsError = document.getElementById('settings-error'); function updateApiControls() { const decision = apiTypeInput.value === 'systemone'; + updateBatchControls(); document.querySelectorAll('.systemone-only').forEach(el => el.classList.toggle('is-hidden', !decision)); for (const id of ['system-instructions', 'openai-organization', 'openai-project', ...Object.keys(DEFAULT_AI_PARAMS)]) { document.getElementById(id)?.closest('.field')?.classList.toggle('is-hidden', decision); @@ -1097,6 +1108,14 @@ document.addEventListener('DOMContentLoaded', async () => { document.getElementById('save').addEventListener('click', async () => { settingsError.textContent = ''; const apiType = apiTypeInput.value; + const systemOneProcessAll = processAllInput.checked; + const enteredBatchSize = Number(batchSizeInput.value); + const validBatchSize = Number.isSafeInteger(enteredBatchSize) && enteredBatchSize > 0; + if (!batchSizeInput.disabled && !validBatchSize) { + settingsError.textContent = 'Rules per request must be a positive whole number.'; + return; + } + const systemOneBatchSize = validBatchSize ? enteredBatchSize : savedBatchSize; const decisionThreshold = thresholdInput.value.trim() === '' ? NaN : Number(thresholdInput.value) / 100; const inputCharacterLimit = Number(inputLimit.value); const endpoint = endpointInput.value.trim(); @@ -1127,13 +1146,15 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - try { validateSettings({ apiType, decisionThreshold, inputCharacterLimit, aiRules: rules }); } + try { validateSettings({ systemOneProcessAll, systemOneBatchSize, apiType, decisionThreshold, inputCharacterLimit, aiRules: rules }); } catch (error) { settingsError.textContent = error.message; return; } - await storage.local.set({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.set({ systemOneProcessAll, systemOneBatchSize, apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + savedBatchSize = systemOneBatchSize; + batchSizeInput.value = savedBatchSize; await storage.local.remove(['templateName', 'customTemplate']); await applyTheme(theme); try { - await AiClassifier.setConfig({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ systemOneProcessAll, systemOneBatchSize, apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); diff --git a/tests/batching.test.mjs b/tests/batching.test.mjs new file mode 100644 index 0000000..000ebe3 --- /dev/null +++ b/tests/batching.test.mjs @@ -0,0 +1,219 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import { readFile } from 'node:fs/promises'; +import { CONFIG_DEFAULTS, validateSettings } from '../modules/classificationConfig.js'; +import * as systemone from '../modules/backends/systemOne.js'; + +const data = {}; +let cacheWrites = 0; +globalThis.messenger = { storage: { local: { + async get(keys) { return Object.fromEntries((Array.isArray(keys) ? keys : [keys]).map(key => [key, data[key]])); }, + async set(values) { Object.assign(data, structuredClone(values)); if ('aiCache' in values) cacheWrites++; }, + async remove() {} +} }, messages: { async get(id) { return { headerMessageId: `mail-${id}` }; } } }; +const classifier = await import('../modules/AiClassifier.js'); +const transfer = await import('../options/dataTransfer.js'); +let requests = []; +async function setup(update = {}) { + await classifier.clearCache(); + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', ...update }); + requests = []; cacheWrites = 0; + globalThis.fetch = async (url, request) => { + const body = JSON.parse(request.body); + requests.push({ url, body, headers: request.headers }); + // Intentionally reverse response order and include an unsolicited answer. + return { ok: true, json: async () => ({ model: 'served', answers: { + unsolicited: { type: 'noul', noul: 1 }, + ...Object.fromEntries(Object.keys(body.questions).reverse().map(id => [id, { type: 'noul', noul: 0.75 }])) + } }) }; + }; + return classifier.getConfig(); +} +async function candidates(count, snapshot, criteria = []) { + return Promise.all(Array.from({ length: count }, async (_, i) => { + const criterion = criteria[i] ?? `criterion ${i}`; + return { id: `rule_${i}`, rule: { criterion, actions: [] }, cacheKey: await classifier.buildCacheKey(1, criterion, snapshot) }; + })); +} +async function collect(items, snapshot) { + const results = []; + for await (const value of classifier.classifyRules('email state', items, snapshot)) results.push(value); + return results; +} + +test('Process all, explicit batch sizes, partial batches, empty input and cache reuse', async () => { + for (const [settings, expected] of [ + [{}, [10]], [{ systemOneProcessAll: false, systemOneBatchSize: 8 }, [8, 2]], + [{ systemOneProcessAll: false, systemOneBatchSize: 1 }, Array(10).fill(1)] + ]) { + const snapshot = await setup(settings); + const items = await candidates(10, snapshot); + const results = await collect(items, snapshot); + assert.deepEqual(requests.map(r => Object.keys(r.body.questions).length), expected); + assert.equal(cacheWrites, expected.length); + assert.deepEqual(results.map(r => r.rule.criterion), items.map(r => r.rule.criterion)); + assert.ok(results.every(r => r.result.probability === 0.75)); + assert.ok(requests.every(r => r.body.state === 'email state' && !('messages' in r.body))); + await collect(items, snapshot); + assert.equal(requests.length, expected.length); + await collect([], snapshot); + assert.equal(requests.length, expected.length); + } +}); + +test('cached rules consume no capacity and duplicate criteria use independent thresholds', async () => { + const snapshot = await setup({ systemOneProcessAll: false, systemOneBatchSize: 2 }); + const items = await candidates(5, snapshot, ['cached', 'same', 'cached', 'other', 'same']); + await collect([items[0]], snapshot); + requests = []; cacheWrites = 0; + items[1].rule.decisionThreshold = 0.8; + items[4].rule.decisionThreshold = 0.7; + const results = await collect(items, snapshot); + assert.equal(requests.length, 1); + assert.deepEqual(Object.keys(requests[0].body.questions), ['rule_1', 'rule_3']); + assert.equal(results[1].result.matched, false); + assert.equal(results[4].result.matched, true); + assert.equal(cacheWrites, 1); + assert.equal(await classifier.buildCacheKey(1, 'same', { ...snapshot, systemOneProcessAll: true, systemOneBatchSize: 100 }), items[1].cacheKey); +}); + +test('answer identity determines probability, not answer order', () => { + const result = { answers: { rule_2: { type: 'noul', noul: 0 }, rule_1: { type: 'noul', noul: 1 } } }; + assert.deepEqual(systemone.parseBatchResponse(result, [{ id: 'rule_1' }, { id: 'rule_2' }]).map(r => r.probability), [1, 0]); +}); + +test('failed batches neither yield nor cache partial results and do not retry', async () => { + for (const failure of ['missing', 'invalid', 'http']) { + const snapshot = await setup(); + const items = await candidates(2, snapshot); + let calls = 0, yielded = 0; + globalThis.fetch = async () => { + calls++; + return { ok: failure !== 'http', status: 413, json: async () => ({ answers: { + rule_0: { type: 'noul', noul: 0.9 }, + ...(failure === 'invalid' ? { rule_1: { type: 'noul', noul: '0.9' } } : {}) + } }) }; + }; + await assert.rejects(async () => { + for await (const value of classifier.classifyRules('state', items, snapshot)) yielded++; + }); + assert.equal(yielded, 0); + assert.equal(calls, 1); + assert.equal(cacheWrites, 0); + assert.equal(await classifier.getCacheSize(), 0); + } +}); + +test('clearing or importing during a batch prevents stale cache writes', async () => { + for (const replace of [() => classifier.clearCache(), () => classifier.replaceCache({})]) { + const snapshot = await setup(); + const items = await candidates(2, snapshot); + const fetch = globalThis.fetch; + globalThis.fetch = async (...args) => { await replace(); return fetch(...args); }; + const results = await collect(items, snapshot); + assert.equal(results.length, 2); + assert.equal(requests.length, 1); + assert.equal(await classifier.getCacheSize(), 0); + } +}); + +test('Chat Completions remains lazy and sequential', async () => { + const snapshot = await setup({ apiType: 'chat-completions' }); + const items = await candidates(3, snapshot); + globalThis.fetch = async (url, request) => { + requests.push(JSON.parse(request.body)); + return { ok: true, json: async () => ({ choices: [{ message: { content: '{"match":true}' } }] }) }; + }; + for await (const value of classifier.classifyRules('state', items, snapshot)) break; + assert.equal(requests.length, 1); + assert.ok(Array.isArray(requests[0].messages)); +}); + +test('batch settings validate and survive import', async () => { + for (const value of [0, -1, 1.2, Infinity, '8', null]) assert.throws(() => validateSettings({ systemOneBatchSize: value })); + assert.throws(() => validateSettings({ systemOneProcessAll: 'true' })); + await transfer.importData({ text: async () => JSON.stringify({ systemOneProcessAll: false, systemOneBatchSize: 3 }) }, ['settings']); + assert.equal(data.systemOneProcessAll, false); + assert.equal(data.systemOneBatchSize, 3); + await assert.rejects(transfer.importData({ text: async () => '{"systemOneBatchSize":0}' }, ['settings'])); +}); + +// Exercise the action runner with the real lazy classifier and mocked mail APIs. +const source = await readFile(new URL('../background.js', import.meta.url), 'utf8'); +const functions = source.slice(source.indexOf('async function processMessage('), source.indexOf('async function clearCacheForMessages(')); +test('stop after match skips later actions and batches; all-mode retains later evidence', async () => { + for (const processAll of [true, false]) { + const snapshot = await setup({ systemOneProcessAll: processAll, systemOneBatchSize: 2 }); + const actions = [], errors = []; + const rules = Array.from({ length: 5 }, (_, i) => ({ criterion: `rule ${i}`, actions: [{ type: 'flag', flagged: true }], stopProcessing: i === 1 })); + const context = vm.createContext({ + structuredClone, Date, Error, aiRules: rules, AiClassifier: classifier, + queue: Promise.resolve(), queuedCount: 0, processing: false, currentStart: 0, + tokenReduction: false, showDebugTab: false, timingStats: {}, + updateActionIcon() {}, updateTimingStats() {}, showTransientIcon() {}, + buildEmailText() { return 'state'; }, ICONS: { circle: 'circle' }, + logger: { aiLog() {} }, recordError(...args) { errors.push(args); }, ERROR_NOTIFICATION_ID: 'error', + storage: messenger.storage, + messenger: { messages: { + async getFull() { return {}; }, async get() { return { tags: [], folder: { accountId: 'account' } }; }, + async update(id, value) { actions.push({ id, value }); } + }, identities: { async list() { return []; } } }, + browser: { runtime: { getURL: x => x }, notifications: { async create() {} } } + }); + vm.runInContext(functions, context); + await vm.runInContext('applyAiRules([1])', context); + assert.equal(errors.length, 0); + assert.equal(actions.length, 2); + assert.equal(requests.length, 1); + assert.equal(Object.keys(requests[0].body.questions).length, processAll ? 5 : 2); + assert.equal(await classifier.getCacheSize(), processAll ? 5 : 2); + } +}); + +test('batch controls default to all and retain numeric input across toggles and API switches', async () => { + const optionsSource = await readFile(new URL('../options/options.js', import.meta.url), 'utf8'); + const section = optionsSource.slice(optionsSource.indexOf(' const apiTypeInput ='), optionsSource.indexOf(' const endpointInput =')); + const fields = new Map(); + const field = id => { + if (!fields.has(id)) fields.set(id, { + value: '', checked: false, disabled: false, listeners: {}, + addEventListener(event, fn) { this.listeners[event] = fn; }, + classList: { toggle() {} }, closest() { return this; } + }); + return fields.get(id); + }; + const context = vm.createContext({ + defaults: { apiType: 'systemone' }, DEFAULT_AI_PARAMS: {}, migratedInputLimit() { return 3686; }, + updateEndpointPreview() {}, document: { getElementById: field, querySelectorAll() { return []; } } + }); + vm.runInContext(`${section}\nupdateApiControls();`, context); + const all = field('systemone-process-all'), size = field('systemone-batch-size'), api = field('api-type'); + assert.equal(all.checked, true); + assert.equal(size.value, 8); + assert.equal(size.disabled, true); + all.checked = false; all.listeners.change(); + assert.equal(size.disabled, false); + size.value = '3'; + all.checked = true; all.listeners.change(); + assert.equal(size.disabled, true); + all.checked = false; all.listeners.change(); + assert.equal(size.value, '3'); + api.value = 'chat-completions'; api.listeners.change(); + assert.equal(size.disabled, true); + api.value = 'systemone'; api.listeners.change(); + assert.equal(size.disabled, false); + assert.equal(size.value, '3'); +}); + +test('batch settings remain fixed while a per-email snapshot is running', async () => { + const snapshot = await setup({ systemOneProcessAll: false, systemOneBatchSize: 2 }); + const items = await candidates(5, snapshot); + const fetch = globalThis.fetch; + globalThis.fetch = async (...args) => { + classifier.setConfig({ systemOneProcessAll: true, systemOneBatchSize: 1 }); + return fetch(...args); + }; + await collect(items, snapshot); + assert.deepEqual(requests.map(r => Object.keys(r.body.questions).length), [2, 2, 1]); +}); diff --git a/tests/cacheRestore.test.mjs b/tests/cacheRestore.test.mjs index f5ca74d..91a8515 100644 --- a/tests/cacheRestore.test.mjs +++ b/tests/cacheRestore.test.mjs @@ -51,7 +51,7 @@ const messageStart = source.indexOf('browser.runtime.onMessage.addListener('); vm.runInContext(source.slice(messageStart, source.indexOf('\n\n // Automatically classify', messageStart)), context); test('export/reinstall/import restores Details immediately with changed Thunderbird IDs', async () => { - const settings = { ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 }; + const settings = { ...CONFIG_DEFAULTS, systemOneProcessAll: false, systemOneBatchSize: 3, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 }; const rules = [{ criterion: 'invoice', decisionThreshold: 0.8, actions: [] }]; await storage.local.set({ ...settings, aiRules: rules }); globalThis.fetch = async () => ({ ok: true, json: async () => ({ answers: { match: { type: 'noul', noul: 0.9 } } }) }); @@ -73,6 +73,8 @@ test('export/reinstall/import restores Details immediately with changed Thunderb assert.equal((await onMessage({ type: 'sortana:getDetails', id: 99 })).results.length, 0); await transfer.importData(exported, ['settings', 'rules', 'cache']); assert.equal((await onMessage({ type: 'sortana:getCacheSize' })).count, 1); + assert.equal(classifier.getConfig().systemOneProcessAll, false); + assert.equal(classifier.getConfig().systemOneBatchSize, 3); const details = await onMessage({ type: 'sortana:getDetails', id: 99 }); assert.equal(details.results.length, 1); assert.equal(details.results[0].criterion, 'invoice'); diff --git a/tests/ruleExecution.test.mjs b/tests/ruleExecution.test.mjs index 153a050..327a14d 100644 --- a/tests/ruleExecution.test.mjs +++ b/tests/ruleExecution.test.mjs @@ -39,10 +39,13 @@ function runner({ fail = false, changeSettings = false } = {}) { AiClassifier: { getConfig() { return structuredClone(currentConfig); }, async buildCacheKey(id, criterion) { return `${id}:${criterion}`; }, - async classify(text, criterion, options) { - evaluated.push({ text, criterion, ...options }); - if (fail && options.cacheKey.startsWith('1:')) throw new Error('inference failed'); - return { matched: true }; + async *classifyRules(text, candidates, snapshot) { + for (const { rule, cacheKey } of candidates) { + evaluated.push({ text, criterion: rule.criterion, cacheKey, snapshot, + threshold: rule.decisionThreshold ?? snapshot.decisionThreshold }); + if (fail && cacheKey.startsWith('1:')) throw new Error('inference failed'); + yield { rule, result: { matched: true } }; + } } } });