Merge pull request #61 from wagesj45/codex/add-option-to-convert-html-to-markdown

Add Markdown conversion option

README.md

@@ -14,6 +14,7 @@ message meets a specified criterion.
 - **Custom system prompts** – tailor the instructions sent to the model for more precise results.
 - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts.
 - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page.
+- **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service.
 - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service.
 - **Automatic rules** – create rules that tag or move new messages based on AI classification.
 - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match.

_locales/en-US/messages.json

@@ -15,4 +15,5 @@
   "template.custom": { "message": "Custom" },
   "options.save": { "message": "Save" },
   "options.debugLogging": { "message": "Enable debug logging" }
+  ,"options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" }
 }

background.js

@@ -22,6 +22,8 @@ let iconTimer = null;
 let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 };
 let currentStart = 0;
 let logGetTiming = true;
+let htmlToMarkdown = false;
+let TurndownService = null;
 
 function setIcon(path) {
     if (browser.browserAction) {
@@ -70,7 +72,17 @@ function collectText(part, bodyParts, attachments) {
         attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`);
     } else if (ct.startsWith("text/html")) {
         const doc = new DOMParser().parseFromString(body, 'text/html');
-        bodyParts.push(replaceInlineBase64(doc.body.textContent || ""));
+        if (htmlToMarkdown && TurndownService) {
+            try {
+                const td = new TurndownService();
+                const md = td.turndown(doc.body.innerHTML || body);
+                bodyParts.push(replaceInlineBase64(`[HTML Body converted to Markdown]\n${md}`));
+            } catch (e) {
+                bodyParts.push(replaceInlineBase64(doc.body.textContent || ""));
+            }
+        } else {
+            bodyParts.push(replaceInlineBase64(doc.body.textContent || ""));
+        }
     } else {
         bodyParts.push(replaceInlineBase64(body));
     }
@@ -213,16 +225,19 @@ async function clearCacheForMessages(idsInput) {
     try {
         AiClassifier = await import(browser.runtime.getURL("modules/AiClassifier.js"));
         logger.aiLog("AiClassifier imported", {debug: true});
+        const td = await import(browser.runtime.getURL("resources/js/turndown.js"));
+        TurndownService = td.default || td.TurndownService;
     } catch (e) {
         console.error("failed to import AiClassifier", e);
         return;
     }
 
     try {
-        const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "aiRules"]);
+        const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "aiRules"]);
         logger.setDebug(store.debugLogging);
         await AiClassifier.setConfig(store);
         await AiClassifier.init();
+        htmlToMarkdown = store.htmlToMarkdown === true;
         const savedStats = await storage.local.get('classifyStats');
         if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') {
             Object.assign(timingStats, savedStats.classifyStats);
@@ -254,6 +269,10 @@ async function clearCacheForMessages(idsInput) {
                 });
                 logger.aiLog("aiRules updated from storage change", {debug: true}, aiRules);
             }
+            if (changes.htmlToMarkdown) {
+                htmlToMarkdown = changes.htmlToMarkdown.newValue === true;
+                logger.aiLog("htmlToMarkdown updated from storage change", {debug: true}, htmlToMarkdown);
+            }
         });
     } catch (err) {
         logger.aiLog("failed to load config", {level: 'error'}, err);

options/options.html

@@ -98,6 +98,11 @@
                             <input type="checkbox" id="debug-logging"> Enable debug logging
                         </label>
                     </div>
+                    <div class="field">
+                        <label class="checkbox">
+                            <input type="checkbox" id="html-to-markdown"> Convert HTML body to Markdown
+                        </label>
+                    </div>
                     <div class="field">
                         <label class="label" for="max_tokens">Max tokens</label>
                         <div class="control">

options/options.js

@@ -9,6 +9,7 @@ document.addEventListener('DOMContentLoaded', async () => {
         'customSystemPrompt',
         'aiParams',
         'debugLogging',
+        'htmlToMarkdown',
         'aiRules',
         'aiCache'
     ]);
@@ -81,6 +82,9 @@ document.addEventListener('DOMContentLoaded', async () => {
     const debugToggle = document.getElementById('debug-logging');
     debugToggle.checked = defaults.debugLogging === true;
 
+    const htmlToggle = document.getElementById('html-to-markdown');
+    htmlToggle.checked = defaults.htmlToMarkdown === true;
+
     const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {});
     for (const [key, val] of Object.entries(aiParams)) {
         const el = document.getElementById(key);
@@ -395,6 +399,7 @@ document.addEventListener('DOMContentLoaded', async () => {
             }
         }
         const debugLogging = debugToggle.checked;
+        const htmlToMarkdown = htmlToggle.checked;
         const rules = [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => {
             const criterion = ruleEl.querySelector('.criterion').value;
             const actions = [...ruleEl.querySelectorAll('.action-row')].map(row => {
@@ -413,7 +418,7 @@ document.addEventListener('DOMContentLoaded', async () => {
             const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked;
             return { criterion, actions, stopProcessing };
         }).filter(r => r.criterion);
-        await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, aiRules: rules });
+        await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, aiRules: rules });
         try {
             await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging });
             logger.setDebug(debugLogging);

resources/js/turndown.js

@@ -972,3 +972,6 @@ var TurndownService = (function () {
   return TurndownService;
 
 }());
+
+export { TurndownService };
+export default TurndownService;