README.md

@@ -16,7 +16,6 @@ message meets a specified criterion.
 - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page.
 - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service.
 - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service.
-- **Debug tab** – view the last request payload and a diff between the unaltered message text and the final prompt.
 - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override.
 - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range.
 - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match.
@@ -141,8 +140,6 @@ uses the following third party libraries:
   - MIT License
 - [turndown v7.2.0](https://github.com/mixmark-io/turndown/tree/v7.2.0)
   - MIT License
-- [diff](https://github.com/google/diff-match-patch/blob/62f2e689f498f9c92dbc588c58750addec9b1654/javascript/diff_match_patch_uncompressed.js)
-  -  Apache-2.0 license
 
 ## License
 

_locales/en-US/messages.json

@@ -18,8 +18,7 @@
   "options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" },
   "options.stripUrlParams": { "message": "Remove URL tracking parameters" },
   "options.altTextImages": { "message": "Replace images with alt text" },
-  "options.collapseWhitespace": { "message": "Collapse long whitespace" },
+  "options.collapseWhitespace": { "message": "Collapse long whitespace" }
-  "options.tokenReduction": { "message": "Aggressive token reduction" }
   ,"action.read": { "message": "read" }
   ,"action.flag": { "message": "flag" }
     ,"action.copy": { "message": "copy" }

ai-filter.sln

@@ -108,7 +108,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-175
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "js", "js", "{21D2A42C-3F85-465C-9141-C106AFD92B68}"
 	ProjectSection(SolutionItems) = preProject
-		resources\js\diff_match_patch_uncompressed.js = resources\js\diff_match_patch_uncompressed.js
 		resources\js\turndown.js = resources\js\turndown.js
 	EndProjectSection
 EndProject

background.js

@@ -26,14 +26,11 @@ let htmlToMarkdown = false;
 let stripUrlParams = false;
 let altTextImages = false;
 let collapseWhitespace = false;
-let tokenReduction = false;
-let maxTokens = 4096;
 let TurndownService = null;
 let userTheme = 'auto';
 let currentTheme = 'light';
 let detectSystemTheme;
 let errorPending = false;
-let showDebugTab = false;
 const ERROR_NOTIFICATION_ID = 'sortana-error';
 
 function normalizeRules(rules) {
@@ -128,16 +125,12 @@ function byteSize(str) {
 }
 
 function replaceInlineBase64(text) {
-    return text.replace(/(?:data:[^;]+;base64,)?[A-Za-z0-9+/=\r\n]{100,}/g,
+    return text.replace(/[A-Za-z0-9+/]{100,}={0,2}/g,
-        m => tokenReduction ? '__BASE64__' : `[base64: ${byteSize(m)} bytes]`);
+        m => `[base64: ${byteSize(m)} bytes]`);
 }
 
 function sanitizeString(text) {
     let t = String(text);
-    if (tokenReduction) {
-        t = t.replace(/<!--.*?-->/gs, '')
-            .replace(/url\([^\)]*\)/gi, 'url(__IMG__)');
-    }
     if (stripUrlParams) {
         t = t.replace(/https?:\/\/[^\s)]+/g, m => {
             const idx = m.indexOf('?');
@@ -145,7 +138,7 @@ function sanitizeString(text) {
         });
     }
     if (collapseWhitespace) {
-        t = t.replace(/[\u200B-\u200D\u2060\s]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n');
+        t = t.replace(/[ \t\u00A0]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n');
     }
     return t;
 }
@@ -164,26 +157,12 @@ function collectText(part, bodyParts, attachments) {
         attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`);
     } else if (ct.startsWith("text/html")) {
         const doc = new DOMParser().parseFromString(body, 'text/html');
-        if (tokenReduction) {
+        if (altTextImages) {
-            doc.querySelectorAll('script,style').forEach(el => el.remove());
+            doc.querySelectorAll('img').forEach(img => {
-            const walker = doc.createTreeWalker(doc, NodeFilter.SHOW_COMMENT);
+                const alt = img.getAttribute('alt') || '';
-            let node;
+                img.replaceWith(doc.createTextNode(alt));
-            while ((node = walker.nextNode())) {
-                node.parentNode.removeChild(node);
-            }
-            doc.querySelectorAll('*').forEach(el => {
-                for (const attr of Array.from(el.attributes)) {
-                    if (!['href','src','alt'].includes(attr.name)) {
-                        el.removeAttribute(attr.name);
-                    }
-                }
             });
         }
-        doc.querySelectorAll('img').forEach(img => {
-            const alt = img.getAttribute('alt') || '';
-            const text = altTextImages ? alt : '__IMG__';
-            img.replaceWith(doc.createTextNode(text));
-        });
         if (stripUrlParams) {
             doc.querySelectorAll('[href]').forEach(a => {
                 const href = a.getAttribute('href');
@@ -210,46 +189,17 @@ function collectText(part, bodyParts, attachments) {
     }
 }
 
-function collectRawText(part, bodyParts, attachments) {
+function buildEmailText(full) {
-    if (part.parts && part.parts.length) {
-        for (const p of part.parts) collectRawText(p, bodyParts, attachments);
-        return;
-    }
-    const ct = (part.contentType || "text/plain").toLowerCase();
-    const cd = (part.headers?.["content-disposition"]?.[0] || "").toLowerCase();
-    const body = String(part.body || "");
-    if (cd.includes("attachment") || !ct.startsWith("text/")) {
-        const nameMatch = /filename\s*=\s*"?([^";]+)/i.exec(cd) || /name\s*=\s*"?([^";]+)/i.exec(part.headers?.["content-type"]?.[0] || "");
-        const name = nameMatch ? nameMatch[1] : "";
-        attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`);
-    } else if (ct.startsWith("text/html")) {
-        const doc = new DOMParser().parseFromString(body, 'text/html');
-        bodyParts.push(doc.body.textContent || "");
-    } else {
-        bodyParts.push(body);
-    }
-}
-
-function buildEmailText(full, applyTransforms = true) {
     const bodyParts = [];
     const attachments = [];
-    const collect = applyTransforms ? collectText : collectRawText;
+    collectText(full, bodyParts, attachments);
-    collect(full, bodyParts, attachments);
     const headers = Object.entries(full.headers || {})
         .map(([k, v]) => `${k}: ${v.join(' ')}`)
         .join('\n');
     const attachInfo = `Attachments: ${attachments.length}` +
         (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : "");
-    let combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim();
+    const combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim();
-    if (applyTransforms && tokenReduction) {
+    return sanitizeString(combined);
-        const seen = new Set();
-        combined = combined.split('\n').filter(l => {
-            if (seen.has(l)) return false;
-            seen.add(l);
-            return true;
-        }).join('\n');
-    }
-    return applyTransforms ? sanitizeString(combined) : combined;
 }
 
 function updateTimingStats(elapsed) {
@@ -283,17 +233,7 @@ async function processMessage(id) {
     updateActionIcon();
     try {
         const full = await messenger.messages.getFull(id);
-        const originalText = buildEmailText(full, false);
+        const text = buildEmailText(full);
-        let text = buildEmailText(full);
-        if (tokenReduction && maxTokens > 0) {
-            const limit = Math.floor(maxTokens * 0.9);
-            if (text.length > limit) {
-                text = text.slice(0, limit);
-            }
-        }
-        if (showDebugTab) {
-            await storage.local.set({ lastFullText: originalText, lastPromptText: text });
-        }
         let hdr;
         let currentTags = [];
         let alreadyRead = false;
@@ -451,7 +391,7 @@ async function clearCacheForMessages(idsInput) {
     }
 
     try {
-        const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending", "showDebugTab"]);
+        const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "aiRules", "theme", "errorPending"]);
         logger.setDebug(store.debugLogging);
         await AiClassifier.setConfig(store);
         userTheme = store.theme || 'auto';
@@ -461,12 +401,7 @@ async function clearCacheForMessages(idsInput) {
         stripUrlParams = store.stripUrlParams === true;
         altTextImages = store.altTextImages === true;
         collapseWhitespace = store.collapseWhitespace === true;
-        tokenReduction = store.tokenReduction === true;
-        if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') {
-            maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens;
-        }
         errorPending = store.errorPending === true;
-        showDebugTab = store.showDebugTab === true;
         const savedStats = await storage.local.get('classifyStats');
         if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') {
             Object.assign(timingStats, savedStats.classifyStats);
@@ -482,25 +417,6 @@ async function clearCacheForMessages(idsInput) {
                 aiRules = normalizeRules(newRules);
                 logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules);
             }
-            if (changes.endpoint || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) {
-                const config = {};
-                if (changes.endpoint) config.endpoint = changes.endpoint.newValue;
-                if (changes.templateName) config.templateName = changes.templateName.newValue;
-                if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue;
-                if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue;
-                if (changes.aiParams) {
-                    config.aiParams = changes.aiParams.newValue;
-                    if (changes.aiParams.newValue && typeof changes.aiParams.newValue.max_tokens !== 'undefined') {
-                        maxTokens = parseInt(changes.aiParams.newValue.max_tokens) || maxTokens;
-                    }
-                }
-                if (changes.debugLogging) {
-                    config.debugLogging = changes.debugLogging.newValue === true;
-                    logger.setDebug(config.debugLogging);
-                }
-                await AiClassifier.setConfig(config);
-                logger.aiLog("AiClassifier config updated from storage change", { debug: true }, config);
-            }
             if (changes.htmlToMarkdown) {
                 htmlToMarkdown = changes.htmlToMarkdown.newValue === true;
                 logger.aiLog("htmlToMarkdown updated from storage change", { debug: true }, htmlToMarkdown);
@@ -517,13 +433,6 @@ async function clearCacheForMessages(idsInput) {
                 collapseWhitespace = changes.collapseWhitespace.newValue === true;
                 logger.aiLog("collapseWhitespace updated from storage change", { debug: true }, collapseWhitespace);
             }
-            if (changes.tokenReduction) {
-                tokenReduction = changes.tokenReduction.newValue === true;
-                logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction);
-            }
-            if (changes.showDebugTab) {
-                showDebugTab = changes.showDebugTab.newValue === true;
-            }
             if (changes.errorPending) {
                 errorPending = changes.errorPending.newValue === true;
                 updateActionIcon();

manifest.json

@@ -1,13 +1,13 @@
 {
   "manifest_version": 2,
   "name": "Sortana",
-  "version": "2.2.0",
+  "version": "2.1.1",
   "default_locale": "en-US",
   "applications": {
     "gecko": {
       "id": "ai-filter@jordanwages",
       "strict_min_version": "128.0",
-      "strict_max_version": "140.*"
+      "strict_max_version": "139.*"
     }
   },
   "icons": {

modules/AiClassifier.js

@@ -308,11 +308,6 @@ async function classifyText(text, criterion, cacheKey = null) {
   }
 
   const payload = buildPayload(text, criterion);
-  try {
-    await storage.local.set({ lastPayload: JSON.parse(payload) });
-  } catch (e) {
-    aiLog('failed to save last payload', { level: 'warn' }, e);
-  }
 
   aiLog(`[AiClassifier] Sending classification request to ${gEndpoint}`, {debug: true});
   aiLog(`[AiClassifier] Classification request payload:`, { debug: true }, payload);

options/options.html

@@ -31,10 +31,6 @@
         .tag {
             --bulma-tag-h: 318;
         }
-        #diff-display {
-            white-space: pre-wrap;
-            font-family: monospace;
-        }
     </style>
 </head>
 <body>
@@ -51,7 +47,6 @@
                             <li class="is-active" data-tab="settings"><a><span class="icon is-small"><img data-icon="settings" data-size="16" src="../resources/img/settings-light-16.png" alt=""></span><span>Settings</span></a></li>
                             <li data-tab="rules"><a><span class="icon is-small"><img data-icon="clipboarddata" data-size="16" src="../resources/img/clipboarddata-light-16.png" alt=""></span><span>Rules</span></a></li>
                             <li data-tab="maintenance"><a><span class="icon is-small"><img data-icon="gear" data-size="16" src="../resources/img/gear-light-16.png" alt=""></span><span>Maintenance</span></a></li>
-                            <li id="debug-tab-button" class="is-hidden" data-tab="debug"><a><span class="icon is-small"><img data-icon="average" data-size="16" src="../resources/img/average-light-16.png" alt=""></span><span>Debug</span></a></li>
                         </ul>
                     </div>
                 </div>
@@ -149,16 +144,6 @@
                             <input type="checkbox" id="collapse-whitespace"> Collapse long whitespace
                         </label>
                     </div>
-                    <div class="field">
-                        <label class="checkbox">
-                            <input type="checkbox" id="token-reduction"> Aggressive token reduction
-                        </label>
-                    </div>
-                    <div class="field">
-                        <label class="checkbox">
-                            <input type="checkbox" id="show-debug-tab"> Show debug information
-                        </label>
-                    </div>
                     <div class="field">
                         <label class="label" for="max_tokens">Max tokens</label>
                         <div class="control">
@@ -283,21 +268,8 @@
                     </p>
                 </div>
             </div>
-
-            <div id="debug-tab" class="tab-content is-hidden">
-                <h2 class="title is-4">
-                    <span class="icon is-small"><img data-icon="average" data-size="16" src="../resources/img/average-light-16.png" alt=""></span>
-                    <span>Debug</span>
-                </h2>
-                <pre id="payload-display"></pre>
-                <div id="diff-container" class="mt-4 is-hidden">
-                    <label class="label">Prompt diff</label>
-                    <div id="diff-display" class="box content is-family-monospace"></div>
-                </div>
-            </div>
         </div>
     </section>
-    <script src="../resources/js/diff_match_patch_uncompressed.js"></script>
     <script src="options.js"></script>
 </body>
 </html>

options/options.js

@@ -16,14 +16,9 @@ document.addEventListener('DOMContentLoaded', async () => {
         'stripUrlParams',
         'altTextImages',
         'collapseWhitespace',
-        'tokenReduction',
         'aiRules',
         'aiCache',
-        'theme',
+        'theme'
-        'showDebugTab',
-        'lastPayload',
-        'lastFullText',
-        'lastPromptText'
     ]);
     const tabButtons = document.querySelectorAll('#main-tabs li');
     const tabs = document.querySelectorAll('.tab-content');
@@ -68,33 +63,6 @@ document.addEventListener('DOMContentLoaded', async () => {
     }
 
     await applyTheme(themeSelect.value);
-    const payloadDisplay = document.getElementById('payload-display');
-    const diffDisplay = document.getElementById('diff-display');
-    const diffContainer = document.getElementById('diff-container');
-
-    let lastFullText = defaults.lastFullText || '';
-    let lastPromptText = defaults.lastPromptText || '';
-    let lastPayload = defaults.lastPayload ? JSON.stringify(defaults.lastPayload, null, 2) : '';
-
-    if (lastPayload) {
-        payloadDisplay.textContent = lastPayload;
-    }
-    if (lastFullText && lastPromptText && diff_match_patch) {
-        const dmp = new diff_match_patch();
-        dmp.Diff_EditCost = 4;
-        const diffs = dmp.diff_main(lastFullText, lastPromptText);
-        dmp.diff_cleanupEfficiency(diffs);
-        const hasDiff = diffs.some(d => d[0] !== 0);
-        if (hasDiff) {
-            diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
-            diffContainer.classList.remove('is-hidden');
-        } else {
-            diffDisplay.innerHTML = '';
-            diffContainer.classList.add('is-hidden');
-        }
-    } else {
-        diffContainer.classList.add('is-hidden');
-    }
     themeSelect.addEventListener('change', async () => {
         markDirty();
         await applyTheme(themeSelect.value);
@@ -147,20 +115,6 @@ document.addEventListener('DOMContentLoaded', async () => {
     const collapseWhitespaceToggle = document.getElementById('collapse-whitespace');
     collapseWhitespaceToggle.checked = defaults.collapseWhitespace === true;
 
-    const tokenReductionToggle = document.getElementById('token-reduction');
-    tokenReductionToggle.checked = defaults.tokenReduction === true;
-
-    const debugTabToggle = document.getElementById('show-debug-tab');
-    const debugTabBtn = document.getElementById('debug-tab-button');
-    function updateDebugTab() {
-        const visible = debugTabToggle.checked;
-        debugTabBtn.classList.toggle('is-hidden', !visible);
-    }
-    debugTabToggle.checked = defaults.showDebugTab === true;
-    debugTabToggle.addEventListener('change', () => { updateDebugTab(); markDirty(); });
-    updateDebugTab();
-
-
     const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {});
     for (const [key, val] of Object.entries(aiParams)) {
         const el = document.getElementById(key);
@@ -744,38 +698,6 @@ document.addEventListener('DOMContentLoaded', async () => {
         } catch {
             cacheCountEl.textContent = '?';
         }
-
-        try {
-            if (debugTabToggle.checked) {
-                const latest = await storage.local.get(['lastPayload', 'lastFullText', 'lastPromptText']);
-                const payloadStr = latest.lastPayload ? JSON.stringify(latest.lastPayload, null, 2) : '';
-                if (payloadStr !== lastPayload) {
-                    lastPayload = payloadStr;
-                    payloadDisplay.textContent = payloadStr;
-                }
-                if (latest.lastFullText !== lastFullText || latest.lastPromptText !== lastPromptText) {
-                    lastFullText = latest.lastFullText || '';
-                    lastPromptText = latest.lastPromptText || '';
-                    if (lastFullText && lastPromptText && diff_match_patch) {
-                        const dmp = new diff_match_patch();
-                        dmp.Diff_EditCost = 4;
-                        const diffs = dmp.diff_main(lastFullText, lastPromptText);
-                        dmp.diff_cleanupEfficiency(diffs);
-                        const hasDiff = diffs.some(d => d[0] !== 0);
-                        if (hasDiff) {
-                            diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
-                            diffContainer.classList.remove('is-hidden');
-                        } else {
-                            diffDisplay.innerHTML = '';
-                            diffContainer.classList.add('is-hidden');
-                        }
-                    } else {
-                        diffDisplay.innerHTML = '';
-                        diffContainer.classList.add('is-hidden');
-                    }
-                }
-            }
-        } catch {}
     }
 
     refreshMaintenance();
@@ -869,10 +791,8 @@ document.addEventListener('DOMContentLoaded', async () => {
         const stripUrlParams = stripUrlToggle.checked;
         const altTextImages = altTextToggle.checked;
         const collapseWhitespace = collapseWhitespaceToggle.checked;
-        const tokenReduction = tokenReductionToggle.checked;
-        const showDebugTab = debugTabToggle.checked;
         const theme = themeSelect.value;
-        await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab });
+        await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, aiRules: rules, theme });
         await applyTheme(theme);
         try {
             await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging });