From 57860499504344f0bbb9b0abc2b27c1a04848957 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:32:25 -0500 Subject: [PATCH 01/17] Updated icons --- resources/img/circleslash-dark-16.png | Bin 0 -> 392 bytes resources/img/circleslash-dark-32.png | Bin 0 -> 768 bytes resources/img/circleslash-dark-64.png | Bin 0 -> 1620 bytes resources/img/circleslash-light-16.png | Bin 0 -> 393 bytes resources/img/circleslash-light-32.png | Bin 0 -> 764 bytes resources/img/circleslash-light-64.png | Bin 0 -> 1572 bytes resources/img/plus-dark-16.png | Bin 0 -> 199 bytes resources/img/plus-dark-32.png | Bin 0 -> 301 bytes resources/img/plus-dark-64.png | Bin 0 -> 431 bytes resources/img/plus-light-16.png | Bin 0 -> 215 bytes resources/img/plus-light-32.png | Bin 0 -> 305 bytes resources/img/plus-light-64.png | Bin 0 -> 457 bytes resources/svg/circledot.svg | 4 ++++ 13 files changed, 4 insertions(+) create mode 100644 resources/img/circleslash-dark-16.png create mode 100644 resources/img/circleslash-dark-32.png create mode 100644 resources/img/circleslash-dark-64.png create mode 100644 resources/img/circleslash-light-16.png create mode 100644 resources/img/circleslash-light-32.png create mode 100644 resources/img/circleslash-light-64.png create mode 100644 resources/img/plus-dark-16.png create mode 100644 resources/img/plus-dark-32.png create mode 100644 resources/img/plus-dark-64.png create mode 100644 resources/img/plus-light-16.png create mode 100644 resources/img/plus-light-32.png create mode 100644 resources/img/plus-light-64.png create mode 100644 resources/svg/circledot.svg diff --git a/resources/img/circleslash-dark-16.png b/resources/img/circleslash-dark-16.png new file mode 100644 index 0000000000000000000000000000000000000000..a7057deb4d2f26e80f2726ca830bb866b0621565 GIT binary patch literal 392 zcmV;30eAk1P)Vw@yjI4lW9DSICHG@mc=*buB>(wcb|xVe)F zdV#&X>8b;KoVBx#ZGB@museld8n_`CV<)iOF+!K@GPe@5)Vvtxp8o?5T;ZYI!jak} z{eOVhbB+5YL4|=j`g6d~xYjaKsO>pOC6_s8Q`=72U%1g>N mmeCzc5wV`@j0000v2QVT*%Acsn z8{my5AVQ*Eh~v=ffc~fS=I|MmbWzeNNoW1s zrlcK7Ym&ZaiUzczeLxvl0`_Pudq4%4@;*&tH#6u3CV?*v0Jps+$tG|Y=*dfmM5#WfiE%~`=T$Zf722N%rZ~^#eg1iGRrR^lrap1l2Uk8R;CD3oc_km|XF#~we zA}|w?IFyN`ZywUV4@~3$wprMvUgn6-MC>$ajGuN0d>43O;?`{{F12vl-;yuwN#Nr? z)~cV|#FVIYEE|?t4qP+vuy)zRe~?5Vl+R)pz=JOP^-4@&0I2%Kat`n@;HUTh4P0$t z_sHAq0fUyn4U?XcF6=z$sKL1gsOYbRAwTX*`kdh^8I$x<(lJSYB;As*QesQVIQcKenA7dfR z*~KYSqZXi3;8#ri`$+`)a2Ex*W}?_Z@E%~z$JqdmCJ_*>hYK>(A#efDP24+nEs5zI z_0lf*Jriq|CJl*jZPR*`iM%bD-T69jBDQPfJRFf&#ogyw4%~+byyd3UW+MI^%eX6b z6xWIe+mfGqOs&*{%qH!;6$0T*Oi-(iSa!T7$p&znHXbHc`*3a40_jh)D)0tZ=Le;h yrUl6SIv$XFoSs(w<8)i@arz@musj4hD*pgE9+=@hjn3Tw0000s3V zK~#90<(tcoR7DiVf8z`cQ3e@CGYINJ5s*j_1ZRu}o45Z#FQnjtP=fJig~0Tl$4M|aQXqIyD2b=B?L_tCeFUs8*zTXpKZs_v~* z=e9}W|BRc8W}s*kt$^{sEk!cW(S$iB0SkeCUfa0oa7To3de3rqw0fW^RaptlbF6Tox8_p!KkkMJtlFOL|&T zdkPorsM6S8r5VCiNiQdvqtO9vlHQD9WLVM%lBTB+I#gyz+ESe(g!OqUpiR`DcT4&%Amo6g+a$f?c~+A2U=js<;UV^;n!DS_vHU>wIWljD!=667 z8&yD`he$=udCw=YyxspX`9GGtyoVAxec0(w=u_RGrW^EOM% z+hwL|&kiL?lij|)NX;s0kEA<*p+u*xWat4701tVd4FXGn-y_B!0d_iV9l-N7Jw4j< zY5l1tT0W`UyJdxE{6V*$J7|)}bxzA1OaDv4BWaAJvkr~2q=|MvcL_XBZ2)r*Ci;>$ zx`Exm9M7{sU>Wd71N2J3UZ<@cSY$W2!uELx9cW$fYe0%mvy97-^8 zBj^G45w`qN8ne8OZ#-?IWUQxvGH^WMUU$VtJ0vE8=i|!;4Y`_H@o60;D}T20Pan}K#`UQxR2UP+fk=lR{=*oM0}se zaggU1UR?NiA7x}$0lrVi#R(=Z((P?5@w6QrRdbGz-Jav{nFf!;SsrQAl+bbb)~Ms~ zc}z}@jb6+>eH^rc+mGN$;C83&Ti`Fd!S^Aw&C~W`j0sywc<}RFE1So4d_mEQq*)Q} zmL`CiN4Z}ygQ6hx%#i7y2Wa}f8uvs_W4$Uaz6rHoUHZHA3C*B+}>0z07aNfc%l>b#Xf=!LkOTxgq zh{=Z~eJFE3pS9q#WWMmCQf1dRN;fGps};R5R%AXsS18@h@)$|aNcvRK$CSuZ3`Z0I& zGl4s*?QPXQ!rbDX!i46BfUkisabkE`qZtZ_!pLv4e;36-(F!OWMgKfqIQ|8rXRcaJ SCJyfa0000R>Cj?>Xm>#>AZS??OT!&&c?Wl`%pOX%16>*YJQZTwte& z?cx$++@m!|5S1VCg3Yo*P2nKa+*eAfz-d%o20p_yNG3Qb37&DW0{ATUR~S|XY+zR2 zc^SCq$qXGhj83Npa6e+_4Q_qoK>QX6{sI>SQ|!Zf{R&O8YXsfEw5x*=E>{6Z&NZG& z7kV)ygB8HrVdGJKknUng2LFIxaWgka5o`s?IDGCFvAvKz!R^8**SdRxVKk|FOfx*; nxb9Y3@|@ct8QyZvpG|o`f`EVKz>1ZM00000NkvXXu0mjfAl#rS literal 0 HcmV?d00001 diff --git a/resources/img/circleslash-light-32.png b/resources/img/circleslash-light-32.png new file mode 100644 index 0000000000000000000000000000000000000000..a5cddafe898238f4751f222f4e8f2f968090df48 GIT binary patch literal 764 zcmVz$qXR{S;^b3qUCqb^`G*FbjN)AUxC_HntDU07HEV04@Nl z9%vJI1dIV^jBQcBEozpm&ud^LlK|ib@WFs=0k_hmRQ;>KJLA6#jHVKxk+;cz1J8j% zT5^L^0G2!vJ(&ob^HoD$%S!BJ;l!Jc6@N|mPP4}NSx)39z)OQ$3l*TH)?V$K{iHpD zysFc&ZK|UbCZKLqre1JK0rGN@(JLp`hN#* zN7y~4vtw4QfO{rAV|hnzY{w1H9V?)wAAezXxV$1a_C+mk0j5QS0ztd1&tnm3k)k>0 zCV*GKIp7CyUql*lJ|glHIM&xIz+VCFLxCF6-5-se`9M)u&piCfFEy0C`}urw6;b93dBSruSQLbQIZ;cFfs92f#{pD zfvV3cl9C9bPly$bzED4+Eg_MjgeFEys`4$Mx3}ZN%!TdTvwP?6-MxDoPBIU(JA3AT z{%7XQ?0hJ#H8--{lr?~?B~t>FQcX&!TQbFvb{9}eEdka61HcNP16T&MIz`U`r-9?Z zBrpbiqqRPhD!w!bP)gkiJOw-ftR{X-n*)9XMt~8m^>i`_1ERG~EMdI-DKHD@gwrf= z0O(H?b3%r%1`elSbXXb%RwaSGqAsA6Y5`scUIgkqKbr-91I_>!?XzZ}9k?5)FGJ5e zz^hv83%;0(qyn$^0w*c$t!dzW;2B^g&=8@s0q6vt1wOz8#Zvlz1=dux3ji+w|3-ky z1N(r@KwSkS*IOO?tj@6dt^&`eP5_{QHzI({S#K{-qJKJduK;%2I>PGTks1Nk_;({J z9|s;t+He8KgTO=tj^VTj2=n|LFcj6dQy`%c*cCy-_CyK5MES73_jt7ocQCaBry@ug zOoRZSFC^ey05)W3{CI@%1?TisL;%o&Jk(vSapMc=7ofQk0(OS<`7E1#nDJ8~&)6ueifo)^B`Ba}bjMF6l===-yzFuhIE^Uk*&6#qNq zfMq@eydqTi(L(UNqvK=Y+n0R^kXV0oAsKJ=JSKcU7D)iGL})tGK>b28-s-v;m=U^i zYp|TVSosje3z(m4t!GJDlu~WLAz+p8dIH#_wf@^zpVm5uSvm_!9k4$5z%{mC(DjvX zrEx_m)o$xqBfOpfHfpW^ilO>T;n_ei;}Wn!_~5%36?=;{{xIR5-&l_49r;#xHs404 z-~3GR(2?5Aj|zr5PyG!PFVP>13WDdI!Y3C(z$;PDDM$BNF$gN96wu@}m$N?YtXxSt z;FXYbPQE#q3C4iM6vu>134ypK*7#Av1Ky^X0WVD4J%c)>)Z#*>s=LD(7++F?1YE}a zMFqv}60s#i<6E43UTb~1khw;HHW9;Z@@+{VNO7F^5qBH@;=Eki!yqXSbtURf)Z(ZTvAe=;rWV$eV&tD0;H+q!{ydm<#9er zHVDtg=H-O`>i!O&ri|ZVsx-s%z!CO4eCxb&!iV1rz&9@><1Lmg0)``QIcXxaS9tcE z@NAycNS_aigt^!@fjMbWG!Jyv;!qcYXC-kLMXds9S$ub`H~Rpl5dq77v22Kf`vnDa z!1`MDyu}o@G1BKvfK@aqD475nYuNLPfMWulgQ4FGAkaAsPrhXwhG3t zLmui1X;)eLVgT44QTZJ3A<&V6;hmTnUer-pZ_mdjz=9bp5g7Bpe#{ni?44!IFYOl0 zUVqeK=_|f{J`n;exE{uE>5;$j-?C$ z%P~jT?!fHWC|gWpc5(l}1ho^u5#VdB_4!osrA W9v{7)Jvxj40000N&PEETD8$pnF~p)b`Op9V_ROjcoeg`m zMIN*0u<&p+UfO-Kl+n?EhbN)MZLv19(t!z$x>M|WT=RIDnGff!+Q}e3gLCQSLQRPu o4l)(Y98X(KTX+;ixLFt(1S5r_Qda5p0nKOdboFyt=akR{08x)R?EnA( literal 0 HcmV?d00001 diff --git a/resources/img/plus-dark-32.png b/resources/img/plus-dark-32.png new file mode 100644 index 0000000000000000000000000000000000000000..fdb1a30bedc11e42ccd5fecf2b7f7b04d8d3c2b4 GIT binary patch literal 301 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdz&H|6fVg?4`^&rfc{H*UVP*AeO zHKHUqKdq!Zu_%?Hyu4g5GcUV1Ik6yBFTW^#_B$IXprT!#E{-7)t#7Aqi5FP6;Cv18{sz`&F6o1s!Ouko?Z{m=Zr=g&JJp{|#C zocBY5*4pe>zL%`8I6v6L?wQH4?%+O#zdrld3p}wjSk}$@s&V!Ke|r|T1O^~7m7m z!eoYR4ZhzQuRXf+{lMuTe>msxaB3f5-oS45_`UK5*1C1NNxF@)_4V^UR;G7EFwRTs zGhP1M*MKpHaW=~a)*PV&VjDaeq7SHH6?n&drSItZ=imBypViK@{3c%$yt6z)G_QU6 z^KaYzKHging=HC(_1kvAZ`A-UhPUPiub;eOU65fAUMjq1FRR}Hcjkr0ml@a}tbJG` z{<4nW1E*!E&RJ5qALP_BoqLmx))*zTHni55q_Rb@++@jxxV(N&PEETD8hkw+Op&3M4h*%;@(a^_Q^fX>(nSv;4*%86G* z7$<}<+g@m|h-O&8`YmybDnr;rW{U=93kOm4hcWgk*W?~Y1m&HdJ(KtRnKK;WMZ7%? ze=B<&Y`^N}H9lfL$-{k4(T*`TEjGz<6Z@efAHqI4>dz}rD?7>J5psu#y?}wepyAc` xN$Ynpeb<<$s{3<}-~CNbs_qIU@QU7N=J)pu8FoxUvb>SI#(m^ELi+u`80pbhgnYLz3s!n(?AChcDu(RU`~ z^qCAsDfR`d8MF8coO5Law@;e(^mTTcoB;nLJpsYFOUgaJr)H|ZG&KGmygOj}0jK-w z>pxXGb_6iK(~DW{_u2UZBUHakgV=&XhNy;P*aT)hW9G3F|M7AE-#v%+@z-Vx1}19P zX1&?zoc7O3rF7D^NuTD+M0>`*-^s@k5!?7BUViny;@mnmlX^}bJLx$I9P7?so2Qzq z^~ThSh5bF#Q+9pPmXKwCD;s>%?5^I(?|GKqzB=_f+62(uiEFM>(==4eL6Sc$`9}0>n-gH5UpngYnB_iefH6L0*nR*Pgg&ebxsLQ E09huj2mk;8 literal 0 HcmV?d00001 diff --git a/resources/svg/circledot.svg b/resources/svg/circledot.svg new file mode 100644 index 0000000..e380e09 --- /dev/null +++ b/resources/svg/circledot.svg @@ -0,0 +1,4 @@ + + + + From 20dfba840689221271cd0ac1320a1819479bf926 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:40:03 -0500 Subject: [PATCH 02/17] Add active state icons for rule condition buttons --- README.md | 5 +++-- options/options.js | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7637f85..7436436 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,9 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e reorder them, check *Only apply to unread messages* to skip read mail, set optional minimum or maximum message age limits, select the accounts or folders a rule should apply to. Use the - slashed-circle/check button to disable or re-enable a rule, and - check *Stop after match* to halt further processing. Forward and reply actions + slashed-circle/check button to disable or re-enable a rule. The small + circle buttons for optional conditions show a filled dot when active, and + check *Stop after match* to halt further processing. Forward and reply actions open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the configured rules. diff --git a/options/options.js b/options/options.js index 3c2df3e..1cfd03a 100644 --- a/options/options.js +++ b/options/options.js @@ -286,12 +286,18 @@ document.addEventListener('DOMContentLoaded', async () => { const btn = document.createElement('button'); btn.type = 'button'; btn.className = 'button is-small is-light'; - btn.textContent = label; + const icon = document.createElement('img'); + icon.width = 16; + icon.height = 16; + icon.className = 'mr-1'; + btn.appendChild(icon); + btn.append(label); let active = checkbox ? checkbox.checked : sectionEl && !sectionEl.classList.contains('is-hidden'); function update() { - btn.classList.toggle('is-info', active); + btn.classList.toggle('is-active', active); + icon.src = browser.runtime.getURL(`resources/svg/${active ? 'circledot' : 'circle'}.svg`); if (sectionEl) sectionEl.classList.toggle('is-hidden', !active); if (checkbox) checkbox.checked = active; } From 1432690fa818bde2ddf0db02cf5daec5bb3d18be Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 00:55:44 -0500 Subject: [PATCH 03/17] Fix rule condition toggles and add divider --- options/options.js | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/options/options.js b/options/options.js index 1cfd03a..096f1a8 100644 --- a/options/options.js +++ b/options/options.js @@ -282,7 +282,7 @@ document.addEventListener('DOMContentLoaded', async () => { return row; } - function createConditionButton(label, sectionEl, checkbox) { + function createConditionButton(label, sectionEl, checkbox, clearFn) { const btn = document.createElement('button'); btn.type = 'button'; btn.className = 'button is-small is-light'; @@ -300,6 +300,9 @@ document.addEventListener('DOMContentLoaded', async () => { icon.src = browser.runtime.getURL(`resources/svg/${active ? 'circledot' : 'circle'}.svg`); if (sectionEl) sectionEl.classList.toggle('is-hidden', !active); if (checkbox) checkbox.checked = active; + if (!active && typeof clearFn === 'function') { + clearFn(); + } } btn.addEventListener('click', () => { @@ -523,16 +526,30 @@ document.addEventListener('DOMContentLoaded', async () => { condButtons.appendChild(p); } - addCond(createConditionButton('Stop', null, stopCheck)); - addCond(createConditionButton('Unread', null, unreadCheck)); - addCond(createConditionButton('Age', ageBox)); - addCond(createConditionButton('Accounts', acctBox)); - addCond(createConditionButton('Folders', folderBox)); + addCond(createConditionButton('Stop', null, stopCheck, () => { + stopCheck.checked = false; + })); + addCond(createConditionButton('Unread', null, unreadCheck, () => { + unreadCheck.checked = false; + })); + addCond(createConditionButton('Age', ageBox, null, () => { + minInput.value = ''; + maxInput.value = ''; + })); + addCond(createConditionButton('Accounts', acctBox, null, () => { + for (const opt of acctSel.options) opt.selected = false; + })); + addCond(createConditionButton('Folders', folderBox, null, () => { + for (const opt of folderSel.options) opt.selected = false; + })); const body = document.createElement('div'); body.className = 'message-body'; body.appendChild(actionsContainer); body.appendChild(addAction); + const condDivider = document.createElement('hr'); + condDivider.className = 'mt-3 mb-2'; + body.appendChild(condDivider); body.appendChild(condButtons); body.appendChild(stopLabel); body.appendChild(unreadLabel); @@ -582,7 +599,9 @@ document.addEventListener('DOMContentLoaded', async () => { const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, unreadOnly, stopProcessing, enabled }; + const rule = { criterion, actions, enabled }; + if (unreadOnly) rule.unreadOnly = true; + if (stopProcessing) rule.stopProcessing = true; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; if (accounts.length) rule.accounts = accounts; @@ -760,7 +779,9 @@ document.addEventListener('DOMContentLoaded', async () => { const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, unreadOnly, stopProcessing, enabled }; + const rule = { criterion, actions, enabled }; + if (unreadOnly) rule.unreadOnly = true; + if (stopProcessing) rule.stopProcessing = true; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; if (accounts.length) rule.accounts = accounts; From 32c8a0f4158261ac0035fd83352f9e2deaae6904 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 02:22:44 -0500 Subject: [PATCH 04/17] Update AGENTS.md Adding direction to skip running the build scripts. --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 9b94461..401f962 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ This file provides guidelines for codex agents contributing to the Sortana proje ## Testing -There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. +There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. Do **not** run the `ps1` build script or the SVG processing script. ## Documentation From 598e74f8cb342348b1367cdb22d158e49aa606c8 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 19:18:21 -0500 Subject: [PATCH 05/17] Add notifications permission --- README.md | 1 + manifest.json | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7436436..3853127 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Sortana requests the following Thunderbird permissions: - `accountsRead` – list accounts and folders for move or copy actions. - `menus` – add context menu commands. - `tabs` – open new tabs and query the active tab. +- `notifications` – display error notifications. - `compose` – create reply and forward compose windows for matching rules. ## Thunderbird Add-on Store Disclosures diff --git a/manifest.json b/manifest.json index fe019ab..c3b45a7 100644 --- a/manifest.json +++ b/manifest.json @@ -32,17 +32,18 @@ "page": "options/options.html", "open_in_tab": true }, - "permissions": [ - "storage", - "messagesRead", - "messagesMove", - "messagesUpdate", - "messagesTagsList", - "accountsRead", - "menus", - "scripting", - "tabs", - "theme", - "compose" - ] + "permissions": [ + "storage", + "messagesRead", + "messagesMove", + "messagesUpdate", + "messagesTagsList", + "accountsRead", + "menus", + "notifications", + "scripting", + "tabs", + "theme", + "compose" + ] } From d3b16640b99afcfc6b16027de5448551398ea64d Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 19:43:24 -0500 Subject: [PATCH 06/17] Handle endpoint changes without restart --- background.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/background.js b/background.js index 06683e2..c7579d0 100644 --- a/background.js +++ b/background.js @@ -417,6 +417,20 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } + if (changes.endpoint || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { + const config = {}; + if (changes.endpoint) config.endpoint = changes.endpoint.newValue; + if (changes.templateName) config.templateName = changes.templateName.newValue; + if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; + if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; + if (changes.aiParams) config.aiParams = changes.aiParams.newValue; + if (changes.debugLogging) { + config.debugLogging = changes.debugLogging.newValue === true; + logger.setDebug(config.debugLogging); + } + await AiClassifier.setConfig(config); + logger.aiLog("AiClassifier config updated from storage change", { debug: true }, config); + } if (changes.htmlToMarkdown) { htmlToMarkdown = changes.htmlToMarkdown.newValue === true; logger.aiLog("htmlToMarkdown updated from storage change", { debug: true }, htmlToMarkdown); From f56057c0428c5f5244b6e1de092d7ae12356f008 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Wed, 16 Jul 2025 20:56:55 -0500 Subject: [PATCH 07/17] updated manifest to new version for bug fixes. --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index c3b45a7..a3c9f7c 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.1.1", + "version": "2.1.2", "default_locale": "en-US", "applications": { "gecko": { From 183ca8f355783f43a21555338cae6144190c871e Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 17:31:03 -0500 Subject: [PATCH 08/17] Add token reduction options and context length cap --- _locales/en-US/messages.json | 4 ++- background.js | 64 ++++++++++++++++++++++++++++++------ options/options.html | 11 +++++++ options/options.js | 12 ++++++- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index 983bb86..fb439b9 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -18,7 +18,9 @@ "options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" }, "options.stripUrlParams": { "message": "Remove URL tracking parameters" }, "options.altTextImages": { "message": "Replace images with alt text" }, - "options.collapseWhitespace": { "message": "Collapse long whitespace" } + "options.collapseWhitespace": { "message": "Collapse long whitespace" }, + "options.tokenReduction": { "message": "Aggressive token reduction" }, + "options.contextLength": { "message": "Context length" } ,"action.read": { "message": "read" } ,"action.flag": { "message": "flag" } ,"action.copy": { "message": "copy" } diff --git a/background.js b/background.js index c7579d0..c44bfed 100644 --- a/background.js +++ b/background.js @@ -26,6 +26,8 @@ let htmlToMarkdown = false; let stripUrlParams = false; let altTextImages = false; let collapseWhitespace = false; +let tokenReduction = false; +let contextLength = 16384; let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; @@ -125,12 +127,16 @@ function byteSize(str) { } function replaceInlineBase64(text) { - return text.replace(/[A-Za-z0-9+/]{100,}={0,2}/g, - m => `[base64: ${byteSize(m)} bytes]`); + return text.replace(/(?:data:[^;]+;base64,)?[A-Za-z0-9+/=\r\n]{100,}/g, + m => tokenReduction ? '__BASE64__' : `[base64: ${byteSize(m)} bytes]`); } function sanitizeString(text) { let t = String(text); + if (tokenReduction) { + t = t.replace(//gs, '') + .replace(/url\([^\)]*\)/gi, 'url(__IMG__)'); + } if (stripUrlParams) { t = t.replace(/https?:\/\/[^\s)]+/g, m => { const idx = m.indexOf('?'); @@ -138,7 +144,7 @@ function sanitizeString(text) { }); } if (collapseWhitespace) { - t = t.replace(/[ \t\u00A0]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n'); + t = t.replace(/[\u200B-\u200D\u2060\s]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n'); } return t; } @@ -157,12 +163,26 @@ function collectText(part, bodyParts, attachments) { attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`); } else if (ct.startsWith("text/html")) { const doc = new DOMParser().parseFromString(body, 'text/html'); - if (altTextImages) { - doc.querySelectorAll('img').forEach(img => { - const alt = img.getAttribute('alt') || ''; - img.replaceWith(doc.createTextNode(alt)); + if (tokenReduction) { + doc.querySelectorAll('script,style').forEach(el => el.remove()); + const walker = doc.createTreeWalker(doc, NodeFilter.SHOW_COMMENT); + let node; + while ((node = walker.nextNode())) { + node.parentNode.removeChild(node); + } + doc.querySelectorAll('*').forEach(el => { + for (const attr of Array.from(el.attributes)) { + if (!['href','src','alt'].includes(attr.name)) { + el.removeAttribute(attr.name); + } + } }); } + doc.querySelectorAll('img').forEach(img => { + const alt = img.getAttribute('alt') || ''; + const text = altTextImages ? alt : '__IMG__'; + img.replaceWith(doc.createTextNode(text)); + }); if (stripUrlParams) { doc.querySelectorAll('[href]').forEach(a => { const href = a.getAttribute('href'); @@ -198,7 +218,15 @@ function buildEmailText(full) { .join('\n'); const attachInfo = `Attachments: ${attachments.length}` + (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : ""); - const combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); + let combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); + if (tokenReduction) { + const seen = new Set(); + combined = combined.split('\n').filter(l => { + if (seen.has(l)) return false; + seen.add(l); + return true; + }).join('\n'); + } return sanitizeString(combined); } @@ -233,7 +261,13 @@ async function processMessage(id) { updateActionIcon(); try { const full = await messenger.messages.getFull(id); - const text = buildEmailText(full); + let text = buildEmailText(full); + if (tokenReduction && contextLength > 0) { + const limit = Math.floor(contextLength * 0.9); + if (text.length > limit) { + text = text.slice(0, limit); + } + } let hdr; let currentTags = []; let alreadyRead = false; @@ -391,7 +425,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "aiRules", "theme", "errorPending"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "contextLength", "aiRules", "theme", "errorPending"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -401,6 +435,8 @@ async function clearCacheForMessages(idsInput) { stripUrlParams = store.stripUrlParams === true; altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; + tokenReduction = store.tokenReduction === true; + contextLength = parseInt(store.contextLength) || contextLength; errorPending = store.errorPending === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -447,6 +483,14 @@ async function clearCacheForMessages(idsInput) { collapseWhitespace = changes.collapseWhitespace.newValue === true; logger.aiLog("collapseWhitespace updated from storage change", { debug: true }, collapseWhitespace); } + if (changes.tokenReduction) { + tokenReduction = changes.tokenReduction.newValue === true; + logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction); + } + if (changes.contextLength) { + contextLength = parseInt(changes.contextLength.newValue) || contextLength; + logger.aiLog("contextLength updated from storage change", { debug: true }, contextLength); + } if (changes.errorPending) { errorPending = changes.errorPending.newValue === true; updateActionIcon(); diff --git a/options/options.html b/options/options.html index 57f407c..108d7f9 100644 --- a/options/options.html +++ b/options/options.html @@ -144,6 +144,17 @@ Collapse long whitespace +
+ +
+
+ +
+ +
+
diff --git a/options/options.js b/options/options.js index 096f1a8..f5af1b6 100644 --- a/options/options.js +++ b/options/options.js @@ -16,6 +16,8 @@ document.addEventListener('DOMContentLoaded', async () => { 'stripUrlParams', 'altTextImages', 'collapseWhitespace', + 'tokenReduction', + 'contextLength', 'aiRules', 'aiCache', 'theme' @@ -115,6 +117,12 @@ document.addEventListener('DOMContentLoaded', async () => { const collapseWhitespaceToggle = document.getElementById('collapse-whitespace'); collapseWhitespaceToggle.checked = defaults.collapseWhitespace === true; + const tokenReductionToggle = document.getElementById('token-reduction'); + tokenReductionToggle.checked = defaults.tokenReduction === true; + + const contextLengthInput = document.getElementById('context-length'); + contextLengthInput.value = defaults.contextLength || 16384; + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { const el = document.getElementById(key); @@ -791,8 +799,10 @@ document.addEventListener('DOMContentLoaded', async () => { const stripUrlParams = stripUrlToggle.checked; const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; + const tokenReduction = tokenReductionToggle.checked; + const contextLength = parseInt(contextLengthInput.value) || 0; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, aiRules: rules, theme }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, contextLength, aiRules: rules, theme }); await applyTheme(theme); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); From 91b8d1557322d1fee91b92ed9d3fd0a2387a96f7 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 17:33:59 -0500 Subject: [PATCH 09/17] updated icons --- resources/img/circledot-dark-16.png | Bin 0 -> 413 bytes resources/img/circledot-dark-32.png | Bin 0 -> 828 bytes resources/img/circledot-dark-64.png | Bin 0 -> 1753 bytes resources/img/circledot-light-16.png | Bin 0 -> 408 bytes resources/img/circledot-light-32.png | Bin 0 -> 813 bytes resources/img/circledot-light-64.png | Bin 0 -> 1737 bytes 6 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 resources/img/circledot-dark-16.png create mode 100644 resources/img/circledot-dark-32.png create mode 100644 resources/img/circledot-dark-64.png create mode 100644 resources/img/circledot-light-16.png create mode 100644 resources/img/circledot-light-32.png create mode 100644 resources/img/circledot-light-64.png diff --git a/resources/img/circledot-dark-16.png b/resources/img/circledot-dark-16.png new file mode 100644 index 0000000000000000000000000000000000000000..db602abfd60d10c744105a8849186b1ed8d39651 GIT binary patch literal 413 zcmV;O0b>4%P)13?T$|D*yAkU+^Ufr|1vMlM2tk2Fi$p_Gg=Qbc zdccf!Lo9u1wD$LWum1!k4aO$>KXODLxQ`9ifFhY$ zKnHj<>?OIjaasY`k#s9*Ns`~`lcXC-4M_)*wp}hrnn~(364IXy8@a9~NGb%Z){L+Z zJQ#M9+@^7w@x#dWyy#Cn2k!h9dS;Sr0dj6@JWP!=Fq6z{k^?WmMQS8b+H+t^`Zj0G zZJF#Ta7l%gy^^k(q~P~46*lt`$zv*IR7X=%Zs%7?A5}%)+lquESY;#500000NkvXX Hu0mjfLU^6T literal 0 HcmV?d00001 diff --git a/resources/img/circledot-dark-32.png b/resources/img/circledot-dark-32.png new file mode 100644 index 0000000000000000000000000000000000000000..d29ac166e645083f94860aed3be1122ef452ae70 GIT binary patch literal 828 zcmV-C1H=4@P)==>Ghs4Wr& z1QkU80BE8Kt3skC#!fc-Sj>zZhTZ+XU6=SKC$pLFT;4f*=FCh6(EmhZ4L0qP_Db3+ zX@{T7OIngNE9rYx-hedH1PlQ$fCA-F0LFof-X_s^l0qYJ7WiBy;G&mO$OD5wL#+fh z1J5F-6Tl^)8`xlUbAIkJE=v|qf!1mXbOH0m$!nme(n=!f1*VPt9MGATKx-8IK9H>< zc#sTm6DS%IYcf&lKM!dy0s}P!j%Co07$vHSSoxe;`K>wx4|3P|UA3rqtA%QRRb$%A zNfwy#zVnzZYL$iz%W4WPt@IiH7fKNb+h<}Of(M!OF~-AO);Gj9N&Efp$0-_Ui=<&m zFTLE8)SSZZi649ATKeA%95LzXPFTez-~(&%9%xCh-DUI-*$i~}u_)h!t~ zyc8vMNxGI``%zLjemkU<({Q^Vze!q7aB|R(4<()QzcZ3fO6pIr{q19H2?Rn00;^1d z0}HF0Evr@D4rwI=U;WrFDd(A%!?>hPxB3 z#3-vV30%RQYB{5u^K(~O*=hkJxD&KwoHW%tz{>83vE-!`7H|bu^TWhS6Yd%{M&+m3 zGVmNYkJ)OKBm-5yj@#rOr`xppak?n?IQ>zLvswt$6@LJo{<`ZPn7_sV0000fdnWqBdN_-H z*7~n?_SyUFwfCuz#Qz!H(k75L(oR4h;QBNPRF=WV0AK=82aEtl0)v5mR=ab+Y2XNO z6xaiN1N>D=aup>S18xT90<(cJ6yMW2fFFSM8EN#wE(l3JrB2djNp13vHc6W#)%xV@ z%g`7}UzZ^0Fl>`_w-4?~UxB{BYruS!N+}=sM)4-R&c3>}X0=N>=wkI$Qs0AhgQ-DF8*gpZx0=|#M+D4IG=UGWD z5nZ%h(nd*BC3R2GMJqElHfC℘^*1#rP=nh6+i`B8YTIS|e$Af}q1>grs#DA7NNp z$^=wMdN;!Peo6P0AaGmmmvkV4$LdlgV0nbo_a*f#L-1MYA?Xc;_l70D6EHu-QM06( zWenVwISL<(=J`s%_z)-OB~4B-_^do6smZ~wS<={~3Fs^7M2PXO5PUERu7`A7QgxC9 ztaKRdP`cedOy)SaO43`!CZI-AtHa=mE)Z;(yzSuID(RkL67ZG7*pDjjXkVyQNjm7@ zy|vH;)H;l`t3303Qh7k`>CwqOY@PnHL$?mtUDWq>dGRf#c3|TrS7X;=Xb^Cc+*MgS zFbX*4fnYf>5;#n5A*Kx&idmdpg{ueDlfNY0#D6DvFlDvPmaY<*V>fgqV75c|fd^9M zhKutoTVcNf1Aw#SCgwDz>$G_y)*E;Pco0(n0rSf31hxQ|Jo)JXoC0nP=-PoBFa=lx zYgC-!7(Lf`9HF0-)adY$MWdv8kN!S#_;@sG7N~V(a)+lU@;a~vQxXruU|=n#%;p2z z9lE;Ug`I#AjvMxRATk$NP(b^Iz)TN*zjNqvVzkGApD|NN7HycFo3I*Q98>|nV)hCO z!6{6=F^TV(H;Zcp&S9s~LVf0CP+Gni$oujOzr%yM`l*XH>K z%%W>6`Co*^B5XN45cub+#<7Il=U75c^|(YWmrQrWvD-;BDiGo#g#esLIWkvWu*cFRCyT{# zZ58Txb%fJ{GIMaLgP$PxRF}o7LhKboU`d4Y4oM$M8d-wCM@d?*=%ccW3$AYF_oTSowPfPIWE5gQ*b#Bw1l*+ z!j!RUF~y9B$z9L8a168Wzdsgh3rT;Ns0LoeRCVM2=+KTSRGj5n6}k^5&>_VC@uC%2 zmGNKv22O(R8I&4HUqp0K-`FB)v=8pS2deQh@2l&8Wm2@tj4R`OaxT?xbAvDkl4fIm zt%=*MZL|aXFqNG3z#m@N1yHI41kn#u0IdUtV>VuH%ILHEfx_(K{*EclAH$qU+(oc} vSpt+J0a3{QHhVEG0%<28b)^0Cbn5sQ=z)1={^||<00000NkvXXu0mjf@q#fR literal 0 HcmV?d00001 diff --git a/resources/img/circledot-light-16.png b/resources/img/circledot-light-16.png new file mode 100644 index 0000000000000000000000000000000000000000..4809c5ac4d11094ddd47913c943a9b612fa3dfb2 GIT binary patch literal 408 zcmV;J0cZY+P)adeJhfOC9u$7hjl3OCLSg!>xvHcsl64}9dW575*>N_W} zjO&cW7B-!n(SmuLVlXOCODQ)v!2nhwVy?mp-mpB77WH&E$^CLlc44f~6_#+7vCzV1 zk~7jAT%h}pN8H6(?ZQS5N#_sb$+U5`JxB*RBz;Aae1|>tASF1IA}{IlOoh#6+ppNy z9_7|Dt>Zi&sad8EoMBCCnaU6AJd!QLeMCI<%KHSW>~1$Tknv>z0000$s4rmL2(nZ6CYrb+pe}$T=)$;jVTiKxDfAJP zC>Qa9yn$e%38;{u55OU(TyzZ-!TV|@{Lj&v*TFg01zZ4* z1E=-%DzFYLipcL=fFhD7I80Y-sJ5OxM}D{u$+(S&eKEop8Q7y?@M zB>*@IyorI%0{4I(;E-8c)^}qXCF}DV=*%SmI0GyhkdMHnkdUf>FYw9ue*wCK1OS~e z^5;M)l-%Gr;4#oJBKAZg>7PeoZvcaNi4%kj84*+QoQN;JYrg!+zmgl=rv`U0QGk|O zl&8Q4cxEvz-4E{)xfT(uir0LLN$Q3N7Y zE`!_-ySCJHRdc`X&Ij;IpIy$mvWRS^0L%avfgAqAalmym*PPk8GCtt}etH=`Ij*vk z9A^V6V41y1xI{)J0OyG%VNw~61(Z`8zP-+?O#nv%{2nMr9jL||(X39+9bhAxSB~kV zPB?19(|Z8q{RV$HNk9d^Q3(DpQRHUS0xT-_3UGLj09|<7sGKNJz@)()*fp1+bc$jR zB=)0Tk#-DDrLjm%84$tm=i~TYt!&no z_1%3_YBua0O?d*cchFQv+}72S=2n1#_&?7jzzC@TBfvE2SF<|s78tgrmS#=~f`@ai riocwmVE5&84SzZPlM64efTG7=cGHKb4{C8?00000NkvXXu0mjf>c(M) literal 0 HcmV?d00001 diff --git a/resources/img/circledot-light-64.png b/resources/img/circledot-light-64.png new file mode 100644 index 0000000000000000000000000000000000000000..cb8634a6126b789ca3b9b97553f9b376bd851849 GIT binary patch literal 1737 zcmV;)1~&PLP)n^aOAi z*ah?gKLJ0A$ni|^WnloN)Nr5+m;+28c~9#Heg)P5YeeLb8-zX*5oZPM;sMir@gvnU>GnGsIP+0 zd%yw_ITaUkfKqsUBCv@mQ{o{TX0i!(ETr~La7 z!hZwqb*;Ds$9=%|2plW2GQb-VLO+O#?F@|20K65^8x}b=0D6=Ui@hgnrMLxGXT(_4 zu6Jb;MoLMLYYRF9r1%E)GbIhMK)9pI`p*)Tt4R+NQ+ zm%uzmxnT~YyH1}AR7$DAz(c_O=mZGpkoz9kC?aPa$!P!%05|)pi@;4Ha^g~rF?h@- zIC!jdI5<6nPLPCcsty6&84re!`P-NwFxg_v?~;1KPfDpb(Yb=wh-R(8Dy7sC7czlw zEp=1MzG6&&Z4NxqW#LAMZYsD z1cn>mdqw1;r5ja4k68GgL+2Q7^UBL!ou5Vk52q3AVc%!rTh^##Ek!W#_=|&XxPqW} z=ULj#x3pX4K>jv^XJ1$k7))?oa3-Pg{WQu_XD(7epZb_|s{{GwnL(X56$AO54y%}T znUmiDXVJSTKQ%d!vDZ@fxTS7(%zvw;A;x!6{EICkUQSXTFMAxs;1dL6`z(6kD-DA4 z(1W1aZk7Z2e;Yjit@Sk6;CZszX>gwd*^N{=4em0&>w()tx>hU_kpl2af?wx5n1b&h zcxYR;t2GVKYhhwr9#=&o@+taAEiNqBHPW-)4)NCj8ZuWbUBn~ z+YCMGZ)0wUZvg(Gyk{H%MvA-b@F4VdcsiHo`Mu~(*JcqhSA5x3ghB>I4#d0kK4@62y(cC>7)BEEH?rRD_ zYsDTQB1eF)jOqcvtBL%O4=o@#!+j}6Mi>b80MCN=5R9LGV0zx(jp6;UgON$r_7RM> zSeR@F8uL~%0`qmiP7Cghq5niMYX>e`n5EvMXJ`Q16`leScM2;pTxww`Kxf{{TCjG9 zV0}CCzY0v7(f9C7rWb%2c`NP1`5^WS61_lEVgq>0JJ2^mOy;Mr1S5FLpRwRON5s8= z*aq-~^n`S;3piZ6Cs4jS#Pb5ojm4u1VT(fq3Hn;=pc*RPi2eFbsk|1OSCxWo8QxOa z0Zgrd^3y`5x>8wQ6~6=k7DWi}2R;GDW}tW*`r$;>RoU2{Piz1WOi#$bSSm%}GxWhm zeF~);(7U?zL|>LZ7dPge7{CM9fxd?q2C7t!dc9&U`m8Q=CF#ZXc(3hh@83%Ndx4f3qxftGHc{DIuF6JW zyc6s$nD`7a2_3Jt3zlh10X?ovbcs2uXLBv+3rTa(Pix|KYm;g5MHKQ zJe(YgPJm7UMx!52-hzHOSvn6C^ux)6=s|5auod`0{0Yn~@MUd)Na5bISrPd!Hw<|i f;7Xut{`L6(SmooGdQ@ Date: Sat, 19 Jul 2025 18:42:39 -0500 Subject: [PATCH 10/17] Remove context length option --- _locales/en-US/messages.json | 3 +-- background.js | 23 +++++++++++++---------- options/options.html | 6 ------ options/options.js | 6 +----- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index fb439b9..65181c8 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -19,8 +19,7 @@ "options.stripUrlParams": { "message": "Remove URL tracking parameters" }, "options.altTextImages": { "message": "Replace images with alt text" }, "options.collapseWhitespace": { "message": "Collapse long whitespace" }, - "options.tokenReduction": { "message": "Aggressive token reduction" }, - "options.contextLength": { "message": "Context length" } + "options.tokenReduction": { "message": "Aggressive token reduction" } ,"action.read": { "message": "read" } ,"action.flag": { "message": "flag" } ,"action.copy": { "message": "copy" } diff --git a/background.js b/background.js index c44bfed..b5667e6 100644 --- a/background.js +++ b/background.js @@ -27,7 +27,7 @@ let stripUrlParams = false; let altTextImages = false; let collapseWhitespace = false; let tokenReduction = false; -let contextLength = 16384; +let maxTokens = 4096; let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; @@ -262,8 +262,8 @@ async function processMessage(id) { try { const full = await messenger.messages.getFull(id); let text = buildEmailText(full); - if (tokenReduction && contextLength > 0) { - const limit = Math.floor(contextLength * 0.9); + if (tokenReduction && maxTokens > 0) { + const limit = Math.floor(maxTokens * 0.9); if (text.length > limit) { text = text.slice(0, limit); } @@ -425,7 +425,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "contextLength", "aiRules", "theme", "errorPending"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -436,7 +436,9 @@ async function clearCacheForMessages(idsInput) { altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; tokenReduction = store.tokenReduction === true; - contextLength = parseInt(store.contextLength) || contextLength; + if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') { + maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; + } errorPending = store.errorPending === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -459,7 +461,12 @@ async function clearCacheForMessages(idsInput) { if (changes.templateName) config.templateName = changes.templateName.newValue; if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; - if (changes.aiParams) config.aiParams = changes.aiParams.newValue; + if (changes.aiParams) { + config.aiParams = changes.aiParams.newValue; + if (changes.aiParams.newValue && typeof changes.aiParams.newValue.max_tokens !== 'undefined') { + maxTokens = parseInt(changes.aiParams.newValue.max_tokens) || maxTokens; + } + } if (changes.debugLogging) { config.debugLogging = changes.debugLogging.newValue === true; logger.setDebug(config.debugLogging); @@ -487,10 +494,6 @@ async function clearCacheForMessages(idsInput) { tokenReduction = changes.tokenReduction.newValue === true; logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction); } - if (changes.contextLength) { - contextLength = parseInt(changes.contextLength.newValue) || contextLength; - logger.aiLog("contextLength updated from storage change", { debug: true }, contextLength); - } if (changes.errorPending) { errorPending = changes.errorPending.newValue === true; updateActionIcon(); diff --git a/options/options.html b/options/options.html index 108d7f9..69158b1 100644 --- a/options/options.html +++ b/options/options.html @@ -149,12 +149,6 @@ Aggressive token reduction
-
- -
- -
-
diff --git a/options/options.js b/options/options.js index f5af1b6..ee2914a 100644 --- a/options/options.js +++ b/options/options.js @@ -17,7 +17,6 @@ document.addEventListener('DOMContentLoaded', async () => { 'altTextImages', 'collapseWhitespace', 'tokenReduction', - 'contextLength', 'aiRules', 'aiCache', 'theme' @@ -120,8 +119,6 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReductionToggle = document.getElementById('token-reduction'); tokenReductionToggle.checked = defaults.tokenReduction === true; - const contextLengthInput = document.getElementById('context-length'); - contextLengthInput.value = defaults.contextLength || 16384; const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { @@ -800,9 +797,8 @@ document.addEventListener('DOMContentLoaded', async () => { const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; const tokenReduction = tokenReductionToggle.checked; - const contextLength = parseInt(contextLengthInput.value) || 0; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, contextLength, aiRules: rules, theme }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme }); await applyTheme(theme); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); From db7ce49a5b47ea495681510396d7cbb26ed26343 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:03:37 -0500 Subject: [PATCH 11/17] Add debug tab with payload viewer --- README.md | 1 + modules/AiClassifier.js | 5 +++++ options/options.html | 14 ++++++++++++++ options/options.js | 21 +++++++++++++++++++-- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3853127..43a515b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ message meets a specified criterion. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. +- **Debug tab** – view the last request payload sent to the AI service. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index b35cb2c..8313654 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -308,6 +308,11 @@ async function classifyText(text, criterion, cacheKey = null) { } const payload = buildPayload(text, criterion); + try { + await storage.local.set({ lastPayload: JSON.parse(payload) }); + } catch (e) { + aiLog('failed to save last payload', { level: 'warn' }, e); + } aiLog(`[AiClassifier] Sending classification request to ${gEndpoint}`, {debug: true}); aiLog(`[AiClassifier] Classification request payload:`, { debug: true }, payload); diff --git a/options/options.html b/options/options.html index 69158b1..d14ce99 100644 --- a/options/options.html +++ b/options/options.html @@ -47,6 +47,7 @@
  • Settings
  • Rules
  • Maintenance
    • +
    @@ -215,6 +216,11 @@
    +
    + +
    @@ -273,6 +279,14 @@

    + + diff --git a/options/options.js b/options/options.js index ee2914a..ebd7497 100644 --- a/options/options.js +++ b/options/options.js @@ -19,7 +19,9 @@ document.addEventListener('DOMContentLoaded', async () => { 'tokenReduction', 'aiRules', 'aiCache', - 'theme' + 'theme', + 'showDebugTab', + 'lastPayload' ]); const tabButtons = document.querySelectorAll('#main-tabs li'); const tabs = document.querySelectorAll('.tab-content'); @@ -64,6 +66,10 @@ document.addEventListener('DOMContentLoaded', async () => { } await applyTheme(themeSelect.value); + const payloadDisplay = document.getElementById('payload-display'); + if (defaults.lastPayload) { + payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2); + } themeSelect.addEventListener('change', async () => { markDirty(); await applyTheme(themeSelect.value); @@ -119,6 +125,16 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReductionToggle = document.getElementById('token-reduction'); tokenReductionToggle.checked = defaults.tokenReduction === true; + const debugTabToggle = document.getElementById('show-debug-tab'); + const debugTabBtn = document.getElementById('debug-tab-button'); + function updateDebugTab() { + const visible = debugTabToggle.checked; + debugTabBtn.classList.toggle('is-hidden', !visible); + } + debugTabToggle.checked = defaults.showDebugTab === true; + debugTabToggle.addEventListener('change', () => { updateDebugTab(); markDirty(); }); + updateDebugTab(); + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { @@ -797,8 +813,9 @@ document.addEventListener('DOMContentLoaded', async () => { const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; const tokenReduction = tokenReductionToggle.checked; + const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); await applyTheme(theme); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); From d2de1818ca11956dfa73bb21cddc0feeb4e8c349 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:24:47 -0500 Subject: [PATCH 12/17] Adding diff library --- README.md | 2 + ai-filter.sln | 1 + resources/js/diff_match_patch_uncompressed.js | 2236 +++++++++++++++++ 3 files changed, 2239 insertions(+) create mode 100644 resources/js/diff_match_patch_uncompressed.js diff --git a/README.md b/README.md index 43a515b..64fbf26 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,8 @@ uses the following third party libraries: - MIT License - [turndown v7.2.0](https://github.com/mixmark-io/turndown/tree/v7.2.0) - MIT License +- [diff](https://github.com/google/diff-match-patch/blob/62f2e689f498f9c92dbc588c58750addec9b1654/javascript/diff_match_patch_uncompressed.js) + - Apache-2.0 license ## License diff --git a/ai-filter.sln b/ai-filter.sln index 57705eb..ea71f1b 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -108,6 +108,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-175 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "js", "js", "{21D2A42C-3F85-465C-9141-C106AFD92B68}" ProjectSection(SolutionItems) = preProject + resources\js\diff_match_patch_uncompressed.js = resources\js\diff_match_patch_uncompressed.js resources\js\turndown.js = resources\js\turndown.js EndProjectSection EndProject diff --git a/resources/js/diff_match_patch_uncompressed.js b/resources/js/diff_match_patch_uncompressed.js new file mode 100644 index 0000000..88a702c --- /dev/null +++ b/resources/js/diff_match_patch_uncompressed.js @@ -0,0 +1,2236 @@ +/** + * Diff Match and Patch + * Copyright 2018 The diff-match-patch Authors. + * https://github.com/google/diff-match-patch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Computes the difference between two texts to create a patch. + * Applies the patch onto another text, allowing for errors. + * @author fraser@google.com (Neil Fraser) + */ + +/** + * Class containing the diff, match and patch methods. + * @constructor + */ +var diff_match_patch = function() { + + // Defaults. + // Redefine these in your program to override the defaults. + + // Number of seconds to map a diff before giving up (0 for infinity). + this.Diff_Timeout = 1.0; + // Cost of an empty edit operation in terms of edit characters. + this.Diff_EditCost = 4; + // At what point is no match declared (0.0 = perfection, 1.0 = very loose). + this.Match_Threshold = 0.5; + // How far to search for a match (0 = exact location, 1000+ = broad match). + // A match this many characters away from the expected location will add + // 1.0 to the score (0.0 is a perfect match). + this.Match_Distance = 1000; + // When deleting a large block of text (over ~64 characters), how close do + // the contents have to be to match the expected contents. (0.0 = perfection, + // 1.0 = very loose). Note that Match_Threshold controls how closely the + // end points of a delete need to match. + this.Patch_DeleteThreshold = 0.5; + // Chunk size for context length. + this.Patch_Margin = 4; + + // The number of bits in an int. + this.Match_MaxBits = 32; +}; + + +// DIFF FUNCTIONS + + +/** + * The data structure representing a diff is an array of tuples: + * [[DIFF_DELETE, 'Hello'], [DIFF_INSERT, 'Goodbye'], [DIFF_EQUAL, ' world.']] + * which means: delete 'Hello', add 'Goodbye' and keep ' world.' + */ +var DIFF_DELETE = -1; +var DIFF_INSERT = 1; +var DIFF_EQUAL = 0; + +/** + * Class representing one diff tuple. + * Attempts to look like a two-element array (which is what this used to be). + * @param {number} op Operation, one of: DIFF_DELETE, DIFF_INSERT, DIFF_EQUAL. + * @param {string} text Text to be deleted, inserted, or retained. + * @constructor + */ +diff_match_patch.Diff = function(op, text) { + this[0] = op; + this[1] = text; +}; + +diff_match_patch.Diff.prototype.length = 2; + +/** + * Emulate the output of a two-element array. + * @return {string} Diff operation as a string. + */ +diff_match_patch.Diff.prototype.toString = function() { + return this[0] + ',' + this[1]; +}; + + +/** + * Find the differences between two texts. Simplifies the problem by stripping + * any common prefix or suffix off the texts before diffing. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {boolean=} opt_checklines Optional speedup flag. If present and false, + * then don't run a line-level diff first to identify the changed areas. + * Defaults to true, which does a faster, slightly less optimal diff. + * @param {number=} opt_deadline Optional time when the diff should be complete + * by. Used internally for recursive calls. Users should set DiffTimeout + * instead. + * @return {!Array.} Array of diff tuples. + */ +diff_match_patch.prototype.diff_main = function(text1, text2, opt_checklines, + opt_deadline) { + // Set a deadline by which time the diff must be complete. + if (typeof opt_deadline == 'undefined') { + if (this.Diff_Timeout <= 0) { + opt_deadline = Number.MAX_VALUE; + } else { + opt_deadline = (new Date).getTime() + this.Diff_Timeout * 1000; + } + } + var deadline = opt_deadline; + + // Check for null inputs. + if (text1 == null || text2 == null) { + throw new Error('Null input. (diff_main)'); + } + + // Check for equality (speedup). + if (text1 == text2) { + if (text1) { + return [new diff_match_patch.Diff(DIFF_EQUAL, text1)]; + } + return []; + } + + if (typeof opt_checklines == 'undefined') { + opt_checklines = true; + } + var checklines = opt_checklines; + + // Trim off common prefix (speedup). + var commonlength = this.diff_commonPrefix(text1, text2); + var commonprefix = text1.substring(0, commonlength); + text1 = text1.substring(commonlength); + text2 = text2.substring(commonlength); + + // Trim off common suffix (speedup). + commonlength = this.diff_commonSuffix(text1, text2); + var commonsuffix = text1.substring(text1.length - commonlength); + text1 = text1.substring(0, text1.length - commonlength); + text2 = text2.substring(0, text2.length - commonlength); + + // Compute the diff on the middle block. + var diffs = this.diff_compute_(text1, text2, checklines, deadline); + + // Restore the prefix and suffix. + if (commonprefix) { + diffs.unshift(new diff_match_patch.Diff(DIFF_EQUAL, commonprefix)); + } + if (commonsuffix) { + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, commonsuffix)); + } + this.diff_cleanupMerge(diffs); + return diffs; +}; + + +/** + * Find the differences between two texts. Assumes that the texts do not + * have any common prefix or suffix. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {boolean} checklines Speedup flag. If false, then don't run a + * line-level diff first to identify the changed areas. + * If true, then run a faster, slightly less optimal diff. + * @param {number} deadline Time when the diff should be complete by. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_compute_ = function(text1, text2, checklines, + deadline) { + var diffs; + + if (!text1) { + // Just add some text (speedup). + return [new diff_match_patch.Diff(DIFF_INSERT, text2)]; + } + + if (!text2) { + // Just delete some text (speedup). + return [new diff_match_patch.Diff(DIFF_DELETE, text1)]; + } + + var longtext = text1.length > text2.length ? text1 : text2; + var shorttext = text1.length > text2.length ? text2 : text1; + var i = longtext.indexOf(shorttext); + if (i != -1) { + // Shorter text is inside the longer text (speedup). + diffs = [new diff_match_patch.Diff(DIFF_INSERT, longtext.substring(0, i)), + new diff_match_patch.Diff(DIFF_EQUAL, shorttext), + new diff_match_patch.Diff(DIFF_INSERT, + longtext.substring(i + shorttext.length))]; + // Swap insertions for deletions if diff is reversed. + if (text1.length > text2.length) { + diffs[0][0] = diffs[2][0] = DIFF_DELETE; + } + return diffs; + } + + if (shorttext.length == 1) { + // Single character string. + // After the previous speedup, the character can't be an equality. + return [new diff_match_patch.Diff(DIFF_DELETE, text1), + new diff_match_patch.Diff(DIFF_INSERT, text2)]; + } + + // Check to see if the problem can be split in two. + var hm = this.diff_halfMatch_(text1, text2); + if (hm) { + // A half-match was found, sort out the return data. + var text1_a = hm[0]; + var text1_b = hm[1]; + var text2_a = hm[2]; + var text2_b = hm[3]; + var mid_common = hm[4]; + // Send both pairs off for separate processing. + var diffs_a = this.diff_main(text1_a, text2_a, checklines, deadline); + var diffs_b = this.diff_main(text1_b, text2_b, checklines, deadline); + // Merge the results. + return diffs_a.concat([new diff_match_patch.Diff(DIFF_EQUAL, mid_common)], + diffs_b); + } + + if (checklines && text1.length > 100 && text2.length > 100) { + return this.diff_lineMode_(text1, text2, deadline); + } + + return this.diff_bisect_(text1, text2, deadline); +}; + + +/** + * Do a quick line-level diff on both strings, then rediff the parts for + * greater accuracy. + * This speedup can produce non-minimal diffs. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} deadline Time when the diff should be complete by. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_lineMode_ = function(text1, text2, deadline) { + // Scan the text on a line-by-line basis first. + var a = this.diff_linesToChars_(text1, text2); + text1 = a.chars1; + text2 = a.chars2; + var linearray = a.lineArray; + + var diffs = this.diff_main(text1, text2, false, deadline); + + // Convert the diff back to original text. + this.diff_charsToLines_(diffs, linearray); + // Eliminate freak matches (e.g. blank lines) + this.diff_cleanupSemantic(diffs); + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, '')); + var pointer = 0; + var count_delete = 0; + var count_insert = 0; + var text_delete = ''; + var text_insert = ''; + while (pointer < diffs.length) { + switch (diffs[pointer][0]) { + case DIFF_INSERT: + count_insert++; + text_insert += diffs[pointer][1]; + break; + case DIFF_DELETE: + count_delete++; + text_delete += diffs[pointer][1]; + break; + case DIFF_EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete >= 1 && count_insert >= 1) { + // Delete the offending records and add the merged ones. + diffs.splice(pointer - count_delete - count_insert, + count_delete + count_insert); + pointer = pointer - count_delete - count_insert; + var subDiff = + this.diff_main(text_delete, text_insert, false, deadline); + for (var j = subDiff.length - 1; j >= 0; j--) { + diffs.splice(pointer, 0, subDiff[j]); + } + pointer = pointer + subDiff.length; + } + count_insert = 0; + count_delete = 0; + text_delete = ''; + text_insert = ''; + break; + } + pointer++; + } + diffs.pop(); // Remove the dummy entry at the end. + + return diffs; +}; + + +/** + * Find the 'middle snake' of a diff, split the problem in two + * and return the recursively constructed diff. + * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} deadline Time at which to bail if not yet complete. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_bisect_ = function(text1, text2, deadline) { + // Cache the text lengths to prevent multiple calls. + var text1_length = text1.length; + var text2_length = text2.length; + var max_d = Math.ceil((text1_length + text2_length) / 2); + var v_offset = max_d; + var v_length = 2 * max_d; + var v1 = new Array(v_length); + var v2 = new Array(v_length); + // Setting all elements to -1 is faster in Chrome & Firefox than mixing + // integers and undefined. + for (var x = 0; x < v_length; x++) { + v1[x] = -1; + v2[x] = -1; + } + v1[v_offset + 1] = 0; + v2[v_offset + 1] = 0; + var delta = text1_length - text2_length; + // If the total number of characters is odd, then the front path will collide + // with the reverse path. + var front = (delta % 2 != 0); + // Offsets for start and end of k loop. + // Prevents mapping of space beyond the grid. + var k1start = 0; + var k1end = 0; + var k2start = 0; + var k2end = 0; + for (var d = 0; d < max_d; d++) { + // Bail out if deadline is reached. + if ((new Date()).getTime() > deadline) { + break; + } + + // Walk the front path one step. + for (var k1 = -d + k1start; k1 <= d - k1end; k1 += 2) { + var k1_offset = v_offset + k1; + var x1; + if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) { + x1 = v1[k1_offset + 1]; + } else { + x1 = v1[k1_offset - 1] + 1; + } + var y1 = x1 - k1; + while (x1 < text1_length && y1 < text2_length && + text1.charAt(x1) == text2.charAt(y1)) { + x1++; + y1++; + } + v1[k1_offset] = x1; + if (x1 > text1_length) { + // Ran off the right of the graph. + k1end += 2; + } else if (y1 > text2_length) { + // Ran off the bottom of the graph. + k1start += 2; + } else if (front) { + var k2_offset = v_offset + delta - k1; + if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) { + // Mirror x2 onto top-left coordinate system. + var x2 = text1_length - v2[k2_offset]; + if (x1 >= x2) { + // Overlap detected. + return this.diff_bisectSplit_(text1, text2, x1, y1, deadline); + } + } + } + } + + // Walk the reverse path one step. + for (var k2 = -d + k2start; k2 <= d - k2end; k2 += 2) { + var k2_offset = v_offset + k2; + var x2; + if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) { + x2 = v2[k2_offset + 1]; + } else { + x2 = v2[k2_offset - 1] + 1; + } + var y2 = x2 - k2; + while (x2 < text1_length && y2 < text2_length && + text1.charAt(text1_length - x2 - 1) == + text2.charAt(text2_length - y2 - 1)) { + x2++; + y2++; + } + v2[k2_offset] = x2; + if (x2 > text1_length) { + // Ran off the left of the graph. + k2end += 2; + } else if (y2 > text2_length) { + // Ran off the top of the graph. + k2start += 2; + } else if (!front) { + var k1_offset = v_offset + delta - k2; + if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) { + var x1 = v1[k1_offset]; + var y1 = v_offset + x1 - k1_offset; + // Mirror x2 onto top-left coordinate system. + x2 = text1_length - x2; + if (x1 >= x2) { + // Overlap detected. + return this.diff_bisectSplit_(text1, text2, x1, y1, deadline); + } + } + } + } + } + // Diff took too long and hit the deadline or + // number of diffs equals number of characters, no commonality at all. + return [new diff_match_patch.Diff(DIFF_DELETE, text1), + new diff_match_patch.Diff(DIFF_INSERT, text2)]; +}; + + +/** + * Given the location of the 'middle snake', split the diff in two parts + * and recurse. + * @param {string} text1 Old string to be diffed. + * @param {string} text2 New string to be diffed. + * @param {number} x Index of split point in text1. + * @param {number} y Index of split point in text2. + * @param {number} deadline Time at which to bail if not yet complete. + * @return {!Array.} Array of diff tuples. + * @private + */ +diff_match_patch.prototype.diff_bisectSplit_ = function(text1, text2, x, y, + deadline) { + var text1a = text1.substring(0, x); + var text2a = text2.substring(0, y); + var text1b = text1.substring(x); + var text2b = text2.substring(y); + + // Compute both diffs serially. + var diffs = this.diff_main(text1a, text2a, false, deadline); + var diffsb = this.diff_main(text1b, text2b, false, deadline); + + return diffs.concat(diffsb); +}; + + +/** + * Split two texts into an array of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {{chars1: string, chars2: string, lineArray: !Array.}} + * An object containing the encoded text1, the encoded text2 and + * the array of unique strings. + * The zeroth element of the array of unique strings is intentionally blank. + * @private + */ +diff_match_patch.prototype.diff_linesToChars_ = function(text1, text2) { + var lineArray = []; // e.g. lineArray[4] == 'Hello\n' + var lineHash = {}; // e.g. lineHash['Hello\n'] == 4 + + // '\x00' is a valid character, but various debuggers don't like it. + // So we'll insert a junk entry to avoid generating a null character. + lineArray[0] = ''; + + /** + * Split a text into an array of strings. Reduce the texts to a string of + * hashes where each Unicode character represents one line. + * Modifies linearray and linehash through being a closure. + * @param {string} text String to encode. + * @return {string} Encoded string. + * @private + */ + function diff_linesToCharsMunge_(text) { + var chars = ''; + // Walk the text, pulling out a substring for each line. + // text.split('\n') would would temporarily double our memory footprint. + // Modifying text would create many large strings to garbage collect. + var lineStart = 0; + var lineEnd = -1; + // Keeping our own length variable is faster than looking it up. + var lineArrayLength = lineArray.length; + while (lineEnd < text.length - 1) { + lineEnd = text.indexOf('\n', lineStart); + if (lineEnd == -1) { + lineEnd = text.length - 1; + } + var line = text.substring(lineStart, lineEnd + 1); + + if (lineHash.hasOwnProperty ? lineHash.hasOwnProperty(line) : + (lineHash[line] !== undefined)) { + chars += String.fromCharCode(lineHash[line]); + } else { + if (lineArrayLength == maxLines) { + // Bail out at 65535 because + // String.fromCharCode(65536) == String.fromCharCode(0) + line = text.substring(lineStart); + lineEnd = text.length; + } + chars += String.fromCharCode(lineArrayLength); + lineHash[line] = lineArrayLength; + lineArray[lineArrayLength++] = line; + } + lineStart = lineEnd + 1; + } + return chars; + } + // Allocate 2/3rds of the space for text1, the rest for text2. + var maxLines = 40000; + var chars1 = diff_linesToCharsMunge_(text1); + maxLines = 65535; + var chars2 = diff_linesToCharsMunge_(text2); + return {chars1: chars1, chars2: chars2, lineArray: lineArray}; +}; + + +/** + * Rehydrate the text in a diff from a string of line hashes to real lines of + * text. + * @param {!Array.} diffs Array of diff tuples. + * @param {!Array.} lineArray Array of unique strings. + * @private + */ +diff_match_patch.prototype.diff_charsToLines_ = function(diffs, lineArray) { + for (var i = 0; i < diffs.length; i++) { + var chars = diffs[i][1]; + var text = []; + for (var j = 0; j < chars.length; j++) { + text[j] = lineArray[chars.charCodeAt(j)]; + } + diffs[i][1] = text.join(''); + } +}; + + +/** + * Determine the common prefix of two strings. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the start of each + * string. + */ +diff_match_patch.prototype.diff_commonPrefix = function(text1, text2) { + // Quick check for common null cases. + if (!text1 || !text2 || text1.charAt(0) != text2.charAt(0)) { + return 0; + } + // Binary search. + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + var pointermin = 0; + var pointermax = Math.min(text1.length, text2.length); + var pointermid = pointermax; + var pointerstart = 0; + while (pointermin < pointermid) { + if (text1.substring(pointerstart, pointermid) == + text2.substring(pointerstart, pointermid)) { + pointermin = pointermid; + pointerstart = pointermin; + } else { + pointermax = pointermid; + } + pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin); + } + return pointermid; +}; + + +/** + * Determine the common suffix of two strings. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the end of each string. + */ +diff_match_patch.prototype.diff_commonSuffix = function(text1, text2) { + // Quick check for common null cases. + if (!text1 || !text2 || + text1.charAt(text1.length - 1) != text2.charAt(text2.length - 1)) { + return 0; + } + // Binary search. + // Performance analysis: https://neil.fraser.name/news/2007/10/09/ + var pointermin = 0; + var pointermax = Math.min(text1.length, text2.length); + var pointermid = pointermax; + var pointerend = 0; + while (pointermin < pointermid) { + if (text1.substring(text1.length - pointermid, text1.length - pointerend) == + text2.substring(text2.length - pointermid, text2.length - pointerend)) { + pointermin = pointermid; + pointerend = pointermin; + } else { + pointermax = pointermid; + } + pointermid = Math.floor((pointermax - pointermin) / 2 + pointermin); + } + return pointermid; +}; + + +/** + * Determine if the suffix of one string is the prefix of another. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {number} The number of characters common to the end of the first + * string and the start of the second string. + * @private + */ +diff_match_patch.prototype.diff_commonOverlap_ = function(text1, text2) { + // Cache the text lengths to prevent multiple calls. + var text1_length = text1.length; + var text2_length = text2.length; + // Eliminate the null case. + if (text1_length == 0 || text2_length == 0) { + return 0; + } + // Truncate the longer string. + if (text1_length > text2_length) { + text1 = text1.substring(text1_length - text2_length); + } else if (text1_length < text2_length) { + text2 = text2.substring(0, text1_length); + } + var text_length = Math.min(text1_length, text2_length); + // Quick check for the worst case. + if (text1 == text2) { + return text_length; + } + + // Start by looking for a single character match + // and increase length until no match is found. + // Performance analysis: https://neil.fraser.name/news/2010/11/04/ + var best = 0; + var length = 1; + while (true) { + var pattern = text1.substring(text_length - length); + var found = text2.indexOf(pattern); + if (found == -1) { + return best; + } + length += found; + if (found == 0 || text1.substring(text_length - length) == + text2.substring(0, length)) { + best = length; + length++; + } + } +}; + + +/** + * Do the two texts share a substring which is at least half the length of the + * longer text? + * This speedup can produce non-minimal diffs. + * @param {string} text1 First string. + * @param {string} text2 Second string. + * @return {Array.} Five element Array, containing the prefix of + * text1, the suffix of text1, the prefix of text2, the suffix of + * text2 and the common middle. Or null if there was no match. + * @private + */ +diff_match_patch.prototype.diff_halfMatch_ = function(text1, text2) { + if (this.Diff_Timeout <= 0) { + // Don't risk returning a non-optimal diff if we have unlimited time. + return null; + } + var longtext = text1.length > text2.length ? text1 : text2; + var shorttext = text1.length > text2.length ? text2 : text1; + if (longtext.length < 4 || shorttext.length * 2 < longtext.length) { + return null; // Pointless. + } + var dmp = this; // 'this' becomes 'window' in a closure. + + /** + * Does a substring of shorttext exist within longtext such that the substring + * is at least half the length of longtext? + * Closure, but does not reference any external variables. + * @param {string} longtext Longer string. + * @param {string} shorttext Shorter string. + * @param {number} i Start index of quarter length substring within longtext. + * @return {Array.} Five element Array, containing the prefix of + * longtext, the suffix of longtext, the prefix of shorttext, the suffix + * of shorttext and the common middle. Or null if there was no match. + * @private + */ + function diff_halfMatchI_(longtext, shorttext, i) { + // Start with a 1/4 length substring at position i as a seed. + var seed = longtext.substring(i, i + Math.floor(longtext.length / 4)); + var j = -1; + var best_common = ''; + var best_longtext_a, best_longtext_b, best_shorttext_a, best_shorttext_b; + while ((j = shorttext.indexOf(seed, j + 1)) != -1) { + var prefixLength = dmp.diff_commonPrefix(longtext.substring(i), + shorttext.substring(j)); + var suffixLength = dmp.diff_commonSuffix(longtext.substring(0, i), + shorttext.substring(0, j)); + if (best_common.length < suffixLength + prefixLength) { + best_common = shorttext.substring(j - suffixLength, j) + + shorttext.substring(j, j + prefixLength); + best_longtext_a = longtext.substring(0, i - suffixLength); + best_longtext_b = longtext.substring(i + prefixLength); + best_shorttext_a = shorttext.substring(0, j - suffixLength); + best_shorttext_b = shorttext.substring(j + prefixLength); + } + } + if (best_common.length * 2 >= longtext.length) { + return [best_longtext_a, best_longtext_b, + best_shorttext_a, best_shorttext_b, best_common]; + } else { + return null; + } + } + + // First check if the second quarter is the seed for a half-match. + var hm1 = diff_halfMatchI_(longtext, shorttext, + Math.ceil(longtext.length / 4)); + // Check again based on the third quarter. + var hm2 = diff_halfMatchI_(longtext, shorttext, + Math.ceil(longtext.length / 2)); + var hm; + if (!hm1 && !hm2) { + return null; + } else if (!hm2) { + hm = hm1; + } else if (!hm1) { + hm = hm2; + } else { + // Both matched. Select the longest. + hm = hm1[4].length > hm2[4].length ? hm1 : hm2; + } + + // A half-match was found, sort out the return data. + var text1_a, text1_b, text2_a, text2_b; + if (text1.length > text2.length) { + text1_a = hm[0]; + text1_b = hm[1]; + text2_a = hm[2]; + text2_b = hm[3]; + } else { + text2_a = hm[0]; + text2_b = hm[1]; + text1_a = hm[2]; + text1_b = hm[3]; + } + var mid_common = hm[4]; + return [text1_a, text1_b, text2_a, text2_b, mid_common]; +}; + + +/** + * Reduce the number of edits by eliminating semantically trivial equalities. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupSemantic = function(diffs) { + var changes = false; + var equalities = []; // Stack of indices where equalities are found. + var equalitiesLength = 0; // Keeping our own length var is faster in JS. + /** @type {?string} */ + var lastEquality = null; + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer = 0; // Index of current position. + // Number of characters that changed prior to the equality. + var length_insertions1 = 0; + var length_deletions1 = 0; + // Number of characters that changed after the equality. + var length_insertions2 = 0; + var length_deletions2 = 0; + while (pointer < diffs.length) { + if (diffs[pointer][0] == DIFF_EQUAL) { // Equality found. + equalities[equalitiesLength++] = pointer; + length_insertions1 = length_insertions2; + length_deletions1 = length_deletions2; + length_insertions2 = 0; + length_deletions2 = 0; + lastEquality = diffs[pointer][1]; + } else { // An insertion or deletion. + if (diffs[pointer][0] == DIFF_INSERT) { + length_insertions2 += diffs[pointer][1].length; + } else { + length_deletions2 += diffs[pointer][1].length; + } + // Eliminate an equality that is smaller or equal to the edits on both + // sides of it. + if (lastEquality && (lastEquality.length <= + Math.max(length_insertions1, length_deletions1)) && + (lastEquality.length <= Math.max(length_insertions2, + length_deletions2))) { + // Duplicate record. + diffs.splice(equalities[equalitiesLength - 1], 0, + new diff_match_patch.Diff(DIFF_DELETE, lastEquality)); + // Change second copy to insert. + diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT; + // Throw away the equality we just deleted. + equalitiesLength--; + // Throw away the previous equality (it needs to be reevaluated). + equalitiesLength--; + pointer = equalitiesLength > 0 ? equalities[equalitiesLength - 1] : -1; + length_insertions1 = 0; // Reset the counters. + length_deletions1 = 0; + length_insertions2 = 0; + length_deletions2 = 0; + lastEquality = null; + changes = true; + } + } + pointer++; + } + + // Normalize the diff. + if (changes) { + this.diff_cleanupMerge(diffs); + } + this.diff_cleanupSemanticLossless(diffs); + + // Find any overlaps between deletions and insertions. + // e.g: abcxxxxxxdef + // -> abcxxxdef + // e.g: xxxabcdefxxx + // -> defxxxabc + // Only extract an overlap if it is as big as the edit ahead or behind it. + pointer = 1; + while (pointer < diffs.length) { + if (diffs[pointer - 1][0] == DIFF_DELETE && + diffs[pointer][0] == DIFF_INSERT) { + var deletion = diffs[pointer - 1][1]; + var insertion = diffs[pointer][1]; + var overlap_length1 = this.diff_commonOverlap_(deletion, insertion); + var overlap_length2 = this.diff_commonOverlap_(insertion, deletion); + if (overlap_length1 >= overlap_length2) { + if (overlap_length1 >= deletion.length / 2 || + overlap_length1 >= insertion.length / 2) { + // Overlap found. Insert an equality and trim the surrounding edits. + diffs.splice(pointer, 0, new diff_match_patch.Diff(DIFF_EQUAL, + insertion.substring(0, overlap_length1))); + diffs[pointer - 1][1] = + deletion.substring(0, deletion.length - overlap_length1); + diffs[pointer + 1][1] = insertion.substring(overlap_length1); + pointer++; + } + } else { + if (overlap_length2 >= deletion.length / 2 || + overlap_length2 >= insertion.length / 2) { + // Reverse overlap found. + // Insert an equality and swap and trim the surrounding edits. + diffs.splice(pointer, 0, new diff_match_patch.Diff(DIFF_EQUAL, + deletion.substring(0, overlap_length2))); + diffs[pointer - 1][0] = DIFF_INSERT; + diffs[pointer - 1][1] = + insertion.substring(0, insertion.length - overlap_length2); + diffs[pointer + 1][0] = DIFF_DELETE; + diffs[pointer + 1][1] = + deletion.substring(overlap_length2); + pointer++; + } + } + pointer++; + } + pointer++; + } +}; + + +/** + * Look for single edits surrounded on both sides by equalities + * which can be shifted sideways to align the edit to a word boundary. + * e.g: The cat came. -> The cat came. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupSemanticLossless = function(diffs) { + /** + * Given two strings, compute a score representing whether the internal + * boundary falls on logical boundaries. + * Scores range from 6 (best) to 0 (worst). + * Closure, but does not reference any external variables. + * @param {string} one First string. + * @param {string} two Second string. + * @return {number} The score. + * @private + */ + function diff_cleanupSemanticScore_(one, two) { + if (!one || !two) { + // Edges are the best. + return 6; + } + + // Each port of this function behaves slightly differently due to + // subtle differences in each language's definition of things like + // 'whitespace'. Since this function's purpose is largely cosmetic, + // the choice has been made to use each language's native features + // rather than force total conformity. + var char1 = one.charAt(one.length - 1); + var char2 = two.charAt(0); + var nonAlphaNumeric1 = char1.match(diff_match_patch.nonAlphaNumericRegex_); + var nonAlphaNumeric2 = char2.match(diff_match_patch.nonAlphaNumericRegex_); + var whitespace1 = nonAlphaNumeric1 && + char1.match(diff_match_patch.whitespaceRegex_); + var whitespace2 = nonAlphaNumeric2 && + char2.match(diff_match_patch.whitespaceRegex_); + var lineBreak1 = whitespace1 && + char1.match(diff_match_patch.linebreakRegex_); + var lineBreak2 = whitespace2 && + char2.match(diff_match_patch.linebreakRegex_); + var blankLine1 = lineBreak1 && + one.match(diff_match_patch.blanklineEndRegex_); + var blankLine2 = lineBreak2 && + two.match(diff_match_patch.blanklineStartRegex_); + + if (blankLine1 || blankLine2) { + // Five points for blank lines. + return 5; + } else if (lineBreak1 || lineBreak2) { + // Four points for line breaks. + return 4; + } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) { + // Three points for end of sentences. + return 3; + } else if (whitespace1 || whitespace2) { + // Two points for whitespace. + return 2; + } else if (nonAlphaNumeric1 || nonAlphaNumeric2) { + // One point for non-alphanumeric. + return 1; + } + return 0; + } + + var pointer = 1; + // Intentionally ignore the first and last element (don't need checking). + while (pointer < diffs.length - 1) { + if (diffs[pointer - 1][0] == DIFF_EQUAL && + diffs[pointer + 1][0] == DIFF_EQUAL) { + // This is a single edit surrounded by equalities. + var equality1 = diffs[pointer - 1][1]; + var edit = diffs[pointer][1]; + var equality2 = diffs[pointer + 1][1]; + + // First, shift the edit as far left as possible. + var commonOffset = this.diff_commonSuffix(equality1, edit); + if (commonOffset) { + var commonString = edit.substring(edit.length - commonOffset); + equality1 = equality1.substring(0, equality1.length - commonOffset); + edit = commonString + edit.substring(0, edit.length - commonOffset); + equality2 = commonString + equality2; + } + + // Second, step character by character right, looking for the best fit. + var bestEquality1 = equality1; + var bestEdit = edit; + var bestEquality2 = equality2; + var bestScore = diff_cleanupSemanticScore_(equality1, edit) + + diff_cleanupSemanticScore_(edit, equality2); + while (edit.charAt(0) === equality2.charAt(0)) { + equality1 += edit.charAt(0); + edit = edit.substring(1) + equality2.charAt(0); + equality2 = equality2.substring(1); + var score = diff_cleanupSemanticScore_(equality1, edit) + + diff_cleanupSemanticScore_(edit, equality2); + // The >= encourages trailing rather than leading whitespace on edits. + if (score >= bestScore) { + bestScore = score; + bestEquality1 = equality1; + bestEdit = edit; + bestEquality2 = equality2; + } + } + + if (diffs[pointer - 1][1] != bestEquality1) { + // We have an improvement, save it back to the diff. + if (bestEquality1) { + diffs[pointer - 1][1] = bestEquality1; + } else { + diffs.splice(pointer - 1, 1); + pointer--; + } + diffs[pointer][1] = bestEdit; + if (bestEquality2) { + diffs[pointer + 1][1] = bestEquality2; + } else { + diffs.splice(pointer + 1, 1); + pointer--; + } + } + } + pointer++; + } +}; + +// Define some regex patterns for matching boundaries. +diff_match_patch.nonAlphaNumericRegex_ = /[^a-zA-Z0-9]/; +diff_match_patch.whitespaceRegex_ = /\s/; +diff_match_patch.linebreakRegex_ = /[\r\n]/; +diff_match_patch.blanklineEndRegex_ = /\n\r?\n$/; +diff_match_patch.blanklineStartRegex_ = /^\r?\n\r?\n/; + +/** + * Reduce the number of edits by eliminating operationally trivial equalities. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupEfficiency = function(diffs) { + var changes = false; + var equalities = []; // Stack of indices where equalities are found. + var equalitiesLength = 0; // Keeping our own length var is faster in JS. + /** @type {?string} */ + var lastEquality = null; + // Always equal to diffs[equalities[equalitiesLength - 1]][1] + var pointer = 0; // Index of current position. + // Is there an insertion operation before the last equality. + var pre_ins = false; + // Is there a deletion operation before the last equality. + var pre_del = false; + // Is there an insertion operation after the last equality. + var post_ins = false; + // Is there a deletion operation after the last equality. + var post_del = false; + while (pointer < diffs.length) { + if (diffs[pointer][0] == DIFF_EQUAL) { // Equality found. + if (diffs[pointer][1].length < this.Diff_EditCost && + (post_ins || post_del)) { + // Candidate found. + equalities[equalitiesLength++] = pointer; + pre_ins = post_ins; + pre_del = post_del; + lastEquality = diffs[pointer][1]; + } else { + // Not a candidate, and can never become one. + equalitiesLength = 0; + lastEquality = null; + } + post_ins = post_del = false; + } else { // An insertion or deletion. + if (diffs[pointer][0] == DIFF_DELETE) { + post_del = true; + } else { + post_ins = true; + } + /* + * Five types to be split: + * ABXYCD + * AXCD + * ABXC + * AXCD + * ABXC + */ + if (lastEquality && ((pre_ins && pre_del && post_ins && post_del) || + ((lastEquality.length < this.Diff_EditCost / 2) && + (pre_ins + pre_del + post_ins + post_del) == 3))) { + // Duplicate record. + diffs.splice(equalities[equalitiesLength - 1], 0, + new diff_match_patch.Diff(DIFF_DELETE, lastEquality)); + // Change second copy to insert. + diffs[equalities[equalitiesLength - 1] + 1][0] = DIFF_INSERT; + equalitiesLength--; // Throw away the equality we just deleted; + lastEquality = null; + if (pre_ins && pre_del) { + // No changes made which could affect previous entry, keep going. + post_ins = post_del = true; + equalitiesLength = 0; + } else { + equalitiesLength--; // Throw away the previous equality. + pointer = equalitiesLength > 0 ? + equalities[equalitiesLength - 1] : -1; + post_ins = post_del = false; + } + changes = true; + } + } + pointer++; + } + + if (changes) { + this.diff_cleanupMerge(diffs); + } +}; + + +/** + * Reorder and merge like edit sections. Merge equalities. + * Any edit section can move as long as it doesn't cross an equality. + * @param {!Array.} diffs Array of diff tuples. + */ +diff_match_patch.prototype.diff_cleanupMerge = function(diffs) { + // Add a dummy entry at the end. + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, '')); + var pointer = 0; + var count_delete = 0; + var count_insert = 0; + var text_delete = ''; + var text_insert = ''; + var commonlength; + while (pointer < diffs.length) { + switch (diffs[pointer][0]) { + case DIFF_INSERT: + count_insert++; + text_insert += diffs[pointer][1]; + pointer++; + break; + case DIFF_DELETE: + count_delete++; + text_delete += diffs[pointer][1]; + pointer++; + break; + case DIFF_EQUAL: + // Upon reaching an equality, check for prior redundancies. + if (count_delete + count_insert > 1) { + if (count_delete !== 0 && count_insert !== 0) { + // Factor out any common prefixies. + commonlength = this.diff_commonPrefix(text_insert, text_delete); + if (commonlength !== 0) { + if ((pointer - count_delete - count_insert) > 0 && + diffs[pointer - count_delete - count_insert - 1][0] == + DIFF_EQUAL) { + diffs[pointer - count_delete - count_insert - 1][1] += + text_insert.substring(0, commonlength); + } else { + diffs.splice(0, 0, new diff_match_patch.Diff(DIFF_EQUAL, + text_insert.substring(0, commonlength))); + pointer++; + } + text_insert = text_insert.substring(commonlength); + text_delete = text_delete.substring(commonlength); + } + // Factor out any common suffixies. + commonlength = this.diff_commonSuffix(text_insert, text_delete); + if (commonlength !== 0) { + diffs[pointer][1] = text_insert.substring(text_insert.length - + commonlength) + diffs[pointer][1]; + text_insert = text_insert.substring(0, text_insert.length - + commonlength); + text_delete = text_delete.substring(0, text_delete.length - + commonlength); + } + } + // Delete the offending records and add the merged ones. + pointer -= count_delete + count_insert; + diffs.splice(pointer, count_delete + count_insert); + if (text_delete.length) { + diffs.splice(pointer, 0, + new diff_match_patch.Diff(DIFF_DELETE, text_delete)); + pointer++; + } + if (text_insert.length) { + diffs.splice(pointer, 0, + new diff_match_patch.Diff(DIFF_INSERT, text_insert)); + pointer++; + } + pointer++; + } else if (pointer !== 0 && diffs[pointer - 1][0] == DIFF_EQUAL) { + // Merge this equality with the previous one. + diffs[pointer - 1][1] += diffs[pointer][1]; + diffs.splice(pointer, 1); + } else { + pointer++; + } + count_insert = 0; + count_delete = 0; + text_delete = ''; + text_insert = ''; + break; + } + } + if (diffs[diffs.length - 1][1] === '') { + diffs.pop(); // Remove the dummy entry at the end. + } + + // Second pass: look for single edits surrounded on both sides by equalities + // which can be shifted sideways to eliminate an equality. + // e.g: ABAC -> ABAC + var changes = false; + pointer = 1; + // Intentionally ignore the first and last element (don't need checking). + while (pointer < diffs.length - 1) { + if (diffs[pointer - 1][0] == DIFF_EQUAL && + diffs[pointer + 1][0] == DIFF_EQUAL) { + // This is a single edit surrounded by equalities. + if (diffs[pointer][1].substring(diffs[pointer][1].length - + diffs[pointer - 1][1].length) == diffs[pointer - 1][1]) { + // Shift the edit over the previous equality. + diffs[pointer][1] = diffs[pointer - 1][1] + + diffs[pointer][1].substring(0, diffs[pointer][1].length - + diffs[pointer - 1][1].length); + diffs[pointer + 1][1] = diffs[pointer - 1][1] + diffs[pointer + 1][1]; + diffs.splice(pointer - 1, 1); + changes = true; + } else if (diffs[pointer][1].substring(0, diffs[pointer + 1][1].length) == + diffs[pointer + 1][1]) { + // Shift the edit over the next equality. + diffs[pointer - 1][1] += diffs[pointer + 1][1]; + diffs[pointer][1] = + diffs[pointer][1].substring(diffs[pointer + 1][1].length) + + diffs[pointer + 1][1]; + diffs.splice(pointer + 1, 1); + changes = true; + } + } + pointer++; + } + // If shifts were made, the diff needs reordering and another shift sweep. + if (changes) { + this.diff_cleanupMerge(diffs); + } +}; + + +/** + * loc is a location in text1, compute and return the equivalent location in + * text2. + * e.g. 'The cat' vs 'The big cat', 1->1, 5->8 + * @param {!Array.} diffs Array of diff tuples. + * @param {number} loc Location within text1. + * @return {number} Location within text2. + */ +diff_match_patch.prototype.diff_xIndex = function(diffs, loc) { + var chars1 = 0; + var chars2 = 0; + var last_chars1 = 0; + var last_chars2 = 0; + var x; + for (x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_INSERT) { // Equality or deletion. + chars1 += diffs[x][1].length; + } + if (diffs[x][0] !== DIFF_DELETE) { // Equality or insertion. + chars2 += diffs[x][1].length; + } + if (chars1 > loc) { // Overshot the location. + break; + } + last_chars1 = chars1; + last_chars2 = chars2; + } + // Was the location was deleted? + if (diffs.length != x && diffs[x][0] === DIFF_DELETE) { + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); +}; + + +/** + * Convert a diff array into a pretty HTML report. + * @param {!Array.} diffs Array of diff tuples. + * @return {string} HTML representation. + */ +diff_match_patch.prototype.diff_prettyHtml = function(diffs) { + var html = []; + var pattern_amp = /&/g; + var pattern_lt = //g; + var pattern_para = /\n/g; + for (var x = 0; x < diffs.length; x++) { + var op = diffs[x][0]; // Operation (insert, delete, equal) + var data = diffs[x][1]; // Text of change. + var text = data.replace(pattern_amp, '&').replace(pattern_lt, '<') + .replace(pattern_gt, '>').replace(pattern_para, '¶
    '); + switch (op) { + case DIFF_INSERT: + html[x] = '' + text + ''; + break; + case DIFF_DELETE: + html[x] = '' + text + ''; + break; + case DIFF_EQUAL: + html[x] = '' + text + ''; + break; + } + } + return html.join(''); +}; + + +/** + * Compute and return the source text (all equalities and deletions). + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Source text. + */ +diff_match_patch.prototype.diff_text1 = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_INSERT) { + text[x] = diffs[x][1]; + } + } + return text.join(''); +}; + + +/** + * Compute and return the destination text (all equalities and insertions). + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Destination text. + */ +diff_match_patch.prototype.diff_text2 = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + if (diffs[x][0] !== DIFF_DELETE) { + text[x] = diffs[x][1]; + } + } + return text.join(''); +}; + + +/** + * Compute the Levenshtein distance; the number of inserted, deleted or + * substituted characters. + * @param {!Array.} diffs Array of diff tuples. + * @return {number} Number of changes. + */ +diff_match_patch.prototype.diff_levenshtein = function(diffs) { + var levenshtein = 0; + var insertions = 0; + var deletions = 0; + for (var x = 0; x < diffs.length; x++) { + var op = diffs[x][0]; + var data = diffs[x][1]; + switch (op) { + case DIFF_INSERT: + insertions += data.length; + break; + case DIFF_DELETE: + deletions += data.length; + break; + case DIFF_EQUAL: + // A deletion and an insertion is one substitution. + levenshtein += Math.max(insertions, deletions); + insertions = 0; + deletions = 0; + break; + } + } + levenshtein += Math.max(insertions, deletions); + return levenshtein; +}; + + +/** + * Crush the diff into an encoded string which describes the operations + * required to transform text1 into text2. + * E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. + * Operations are tab-separated. Inserted text is escaped using %xx notation. + * @param {!Array.} diffs Array of diff tuples. + * @return {string} Delta text. + */ +diff_match_patch.prototype.diff_toDelta = function(diffs) { + var text = []; + for (var x = 0; x < diffs.length; x++) { + switch (diffs[x][0]) { + case DIFF_INSERT: + text[x] = '+' + encodeURI(diffs[x][1]); + break; + case DIFF_DELETE: + text[x] = '-' + diffs[x][1].length; + break; + case DIFF_EQUAL: + text[x] = '=' + diffs[x][1].length; + break; + } + } + return text.join('\t').replace(/%20/g, ' '); +}; + + +/** + * Given the original text1, and an encoded string which describes the + * operations required to transform text1 into text2, compute the full diff. + * @param {string} text1 Source string for the diff. + * @param {string} delta Delta text. + * @return {!Array.} Array of diff tuples. + * @throws {!Error} If invalid input. + */ +diff_match_patch.prototype.diff_fromDelta = function(text1, delta) { + var diffs = []; + var diffsLength = 0; // Keeping our own length var is faster in JS. + var pointer = 0; // Cursor in text1 + var tokens = delta.split(/\t/g); + for (var x = 0; x < tokens.length; x++) { + // Each token begins with a one character parameter which specifies the + // operation of this token (delete, insert, equality). + var param = tokens[x].substring(1); + switch (tokens[x].charAt(0)) { + case '+': + try { + diffs[diffsLength++] = + new diff_match_patch.Diff(DIFF_INSERT, decodeURI(param)); + } catch (ex) { + // Malformed URI sequence. + throw new Error('Illegal escape in diff_fromDelta: ' + param); + } + break; + case '-': + // Fall through. + case '=': + var n = parseInt(param, 10); + if (isNaN(n) || n < 0) { + throw new Error('Invalid number in diff_fromDelta: ' + param); + } + var text = text1.substring(pointer, pointer += n); + if (tokens[x].charAt(0) == '=') { + diffs[diffsLength++] = new diff_match_patch.Diff(DIFF_EQUAL, text); + } else { + diffs[diffsLength++] = new diff_match_patch.Diff(DIFF_DELETE, text); + } + break; + default: + // Blank tokens are ok (from a trailing \t). + // Anything else is an error. + if (tokens[x]) { + throw new Error('Invalid diff operation in diff_fromDelta: ' + + tokens[x]); + } + } + } + if (pointer != text1.length) { + throw new Error('Delta length (' + pointer + + ') does not equal source text length (' + text1.length + ').'); + } + return diffs; +}; + + +// MATCH FUNCTIONS + + +/** + * Locate the best instance of 'pattern' in 'text' near 'loc'. + * @param {string} text The text to search. + * @param {string} pattern The pattern to search for. + * @param {number} loc The location to search around. + * @return {number} Best match index or -1. + */ +diff_match_patch.prototype.match_main = function(text, pattern, loc) { + // Check for null inputs. + if (text == null || pattern == null || loc == null) { + throw new Error('Null input. (match_main)'); + } + + loc = Math.max(0, Math.min(loc, text.length)); + if (text == pattern) { + // Shortcut (potentially not guaranteed by the algorithm) + return 0; + } else if (!text.length) { + // Nothing to match. + return -1; + } else if (text.substring(loc, loc + pattern.length) == pattern) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + return this.match_bitap_(text, pattern, loc); + } +}; + + +/** + * Locate the best instance of 'pattern' in 'text' near 'loc' using the + * Bitap algorithm. + * @param {string} text The text to search. + * @param {string} pattern The pattern to search for. + * @param {number} loc The location to search around. + * @return {number} Best match index or -1. + * @private + */ +diff_match_patch.prototype.match_bitap_ = function(text, pattern, loc) { + if (pattern.length > this.Match_MaxBits) { + throw new Error('Pattern too long for this browser.'); + } + + // Initialise the alphabet. + var s = this.match_alphabet_(pattern); + + var dmp = this; // 'this' becomes 'window' in a closure. + + /** + * Compute and return the score for a match with e errors and x location. + * Accesses loc and pattern through being a closure. + * @param {number} e Number of errors in match. + * @param {number} x Location of match. + * @return {number} Overall score for match (0.0 = good, 1.0 = bad). + * @private + */ + function match_bitapScore_(e, x) { + var accuracy = e / pattern.length; + var proximity = Math.abs(loc - x); + if (!dmp.Match_Distance) { + // Dodge divide by zero error. + return proximity ? 1.0 : accuracy; + } + return accuracy + (proximity / dmp.Match_Distance); + } + + // Highest score beyond which we give up. + var score_threshold = this.Match_Threshold; + // Is there a nearby exact match? (speedup) + var best_loc = text.indexOf(pattern, loc); + if (best_loc != -1) { + score_threshold = Math.min(match_bitapScore_(0, best_loc), score_threshold); + // What about in the other direction? (speedup) + best_loc = text.lastIndexOf(pattern, loc + pattern.length); + if (best_loc != -1) { + score_threshold = + Math.min(match_bitapScore_(0, best_loc), score_threshold); + } + } + + // Initialise the bit arrays. + var matchmask = 1 << (pattern.length - 1); + best_loc = -1; + + var bin_min, bin_mid; + var bin_max = pattern.length + text.length; + var last_rd; + for (var d = 0; d < pattern.length; d++) { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at this + // error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (match_bitapScore_(d, loc + bin_mid) <= score_threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = Math.floor((bin_max - bin_min) / 2 + bin_min); + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = Math.max(1, loc - bin_mid + 1); + var finish = Math.min(loc + bin_mid, text.length) + pattern.length; + + var rd = Array(finish + 2); + rd[finish + 1] = (1 << d) - 1; + for (var j = finish; j >= start; j--) { + // The alphabet (s) is a sparse hash, so the following line generates + // warnings. + var charMatch = s[text.charAt(j - 1)]; + if (d === 0) { // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & charMatch; + } else { // Subsequent passes: fuzzy match. + rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) | + (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | + last_rd[j + 1]; + } + if (rd[j] & matchmask) { + var score = match_bitapScore_(d, j - 1); + // This match will almost certainly be better than any existing match. + // But check anyway. + if (score <= score_threshold) { + // Told you so. + score_threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = Math.max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + // No hope for a (better) match at greater error levels. + if (match_bitapScore_(d + 1, loc) > score_threshold) { + break; + } + last_rd = rd; + } + return best_loc; +}; + + +/** + * Initialise the alphabet for the Bitap algorithm. + * @param {string} pattern The text to encode. + * @return {!Object} Hash of character locations. + * @private + */ +diff_match_patch.prototype.match_alphabet_ = function(pattern) { + var s = {}; + for (var i = 0; i < pattern.length; i++) { + s[pattern.charAt(i)] = 0; + } + for (var i = 0; i < pattern.length; i++) { + s[pattern.charAt(i)] |= 1 << (pattern.length - i - 1); + } + return s; +}; + + +// PATCH FUNCTIONS + + +/** + * Increase the context until it is unique, + * but don't let the pattern expand beyond Match_MaxBits. + * @param {!diff_match_patch.patch_obj} patch The patch to grow. + * @param {string} text Source text. + * @private + */ +diff_match_patch.prototype.patch_addContext_ = function(patch, text) { + if (text.length == 0) { + return; + } + if (patch.start2 === null) { + throw Error('patch not initialized'); + } + var pattern = text.substring(patch.start2, patch.start2 + patch.length1); + var padding = 0; + + // Look for the first and last matches of pattern in text. If two different + // matches are found, increase the pattern length. + while (text.indexOf(pattern) != text.lastIndexOf(pattern) && + pattern.length < this.Match_MaxBits - this.Patch_Margin - + this.Patch_Margin) { + padding += this.Patch_Margin; + pattern = text.substring(patch.start2 - padding, + patch.start2 + patch.length1 + padding); + } + // Add one chunk for good luck. + padding += this.Patch_Margin; + + // Add the prefix. + var prefix = text.substring(patch.start2 - padding, patch.start2); + if (prefix) { + patch.diffs.unshift(new diff_match_patch.Diff(DIFF_EQUAL, prefix)); + } + // Add the suffix. + var suffix = text.substring(patch.start2 + patch.length1, + patch.start2 + patch.length1 + padding); + if (suffix) { + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, suffix)); + } + + // Roll back the start points. + patch.start1 -= prefix.length; + patch.start2 -= prefix.length; + // Extend the lengths. + patch.length1 += prefix.length + suffix.length; + patch.length2 += prefix.length + suffix.length; +}; + + +/** + * Compute a list of patches to turn text1 into text2. + * Use diffs if provided, otherwise compute it ourselves. + * There are four ways to call this function, depending on what data is + * available to the caller: + * Method 1: + * a = text1, b = text2 + * Method 2: + * a = diffs + * Method 3 (optimal): + * a = text1, b = diffs + * Method 4 (deprecated, use method 3): + * a = text1, b = text2, c = diffs + * + * @param {string|!Array.} a text1 (methods 1,3,4) or + * Array of diff tuples for text1 to text2 (method 2). + * @param {string|!Array.=} opt_b text2 (methods 1,4) or + * Array of diff tuples for text1 to text2 (method 3) or undefined (method 2). + * @param {string|!Array.=} opt_c Array of diff tuples + * for text1 to text2 (method 4) or undefined (methods 1,2,3). + * @return {!Array.} Array of Patch objects. + */ +diff_match_patch.prototype.patch_make = function(a, opt_b, opt_c) { + var text1, diffs; + if (typeof a == 'string' && typeof opt_b == 'string' && + typeof opt_c == 'undefined') { + // Method 1: text1, text2 + // Compute diffs from text1 and text2. + text1 = /** @type {string} */(a); + diffs = this.diff_main(text1, /** @type {string} */(opt_b), true); + if (diffs.length > 2) { + this.diff_cleanupSemantic(diffs); + this.diff_cleanupEfficiency(diffs); + } + } else if (a && typeof a == 'object' && typeof opt_b == 'undefined' && + typeof opt_c == 'undefined') { + // Method 2: diffs + // Compute text1 from diffs. + diffs = /** @type {!Array.} */(a); + text1 = this.diff_text1(diffs); + } else if (typeof a == 'string' && opt_b && typeof opt_b == 'object' && + typeof opt_c == 'undefined') { + // Method 3: text1, diffs + text1 = /** @type {string} */(a); + diffs = /** @type {!Array.} */(opt_b); + } else if (typeof a == 'string' && typeof opt_b == 'string' && + opt_c && typeof opt_c == 'object') { + // Method 4: text1, text2, diffs + // text2 is not used. + text1 = /** @type {string} */(a); + diffs = /** @type {!Array.} */(opt_c); + } else { + throw new Error('Unknown call format to patch_make.'); + } + + if (diffs.length === 0) { + return []; // Get rid of the null case. + } + var patches = []; + var patch = new diff_match_patch.patch_obj(); + var patchDiffLength = 0; // Keeping our own length var is faster in JS. + var char_count1 = 0; // Number of characters into the text1 string. + var char_count2 = 0; // Number of characters into the text2 string. + // Start with text1 (prepatch_text) and apply the diffs until we arrive at + // text2 (postpatch_text). We recreate the patches one by one to determine + // context info. + var prepatch_text = text1; + var postpatch_text = text1; + for (var x = 0; x < diffs.length; x++) { + var diff_type = diffs[x][0]; + var diff_text = diffs[x][1]; + + if (!patchDiffLength && diff_type !== DIFF_EQUAL) { + // A new patch starts here. + patch.start1 = char_count1; + patch.start2 = char_count2; + } + + switch (diff_type) { + case DIFF_INSERT: + patch.diffs[patchDiffLength++] = diffs[x]; + patch.length2 += diff_text.length; + postpatch_text = postpatch_text.substring(0, char_count2) + diff_text + + postpatch_text.substring(char_count2); + break; + case DIFF_DELETE: + patch.length1 += diff_text.length; + patch.diffs[patchDiffLength++] = diffs[x]; + postpatch_text = postpatch_text.substring(0, char_count2) + + postpatch_text.substring(char_count2 + + diff_text.length); + break; + case DIFF_EQUAL: + if (diff_text.length <= 2 * this.Patch_Margin && + patchDiffLength && diffs.length != x + 1) { + // Small equality inside a patch. + patch.diffs[patchDiffLength++] = diffs[x]; + patch.length1 += diff_text.length; + patch.length2 += diff_text.length; + } else if (diff_text.length >= 2 * this.Patch_Margin) { + // Time for a new patch. + if (patchDiffLength) { + this.patch_addContext_(patch, prepatch_text); + patches.push(patch); + patch = new diff_match_patch.patch_obj(); + patchDiffLength = 0; + // Unlike Unidiff, our patch lists have a rolling context. + // https://github.com/google/diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + prepatch_text = postpatch_text; + char_count1 = char_count2; + } + } + break; + } + + // Update the current character count. + if (diff_type !== DIFF_INSERT) { + char_count1 += diff_text.length; + } + if (diff_type !== DIFF_DELETE) { + char_count2 += diff_text.length; + } + } + // Pick up the leftover patch if not empty. + if (patchDiffLength) { + this.patch_addContext_(patch, prepatch_text); + patches.push(patch); + } + + return patches; +}; + + +/** + * Given an array of patches, return another array that is identical. + * @param {!Array.} patches Array of Patch objects. + * @return {!Array.} Array of Patch objects. + */ +diff_match_patch.prototype.patch_deepCopy = function(patches) { + // Making deep copies is hard in JavaScript. + var patchesCopy = []; + for (var x = 0; x < patches.length; x++) { + var patch = patches[x]; + var patchCopy = new diff_match_patch.patch_obj(); + patchCopy.diffs = []; + for (var y = 0; y < patch.diffs.length; y++) { + patchCopy.diffs[y] = + new diff_match_patch.Diff(patch.diffs[y][0], patch.diffs[y][1]); + } + patchCopy.start1 = patch.start1; + patchCopy.start2 = patch.start2; + patchCopy.length1 = patch.length1; + patchCopy.length2 = patch.length2; + patchesCopy[x] = patchCopy; + } + return patchesCopy; +}; + + +/** + * Merge a set of patches onto the text. Return a patched text, as well + * as a list of true/false values indicating which patches were applied. + * @param {!Array.} patches Array of Patch objects. + * @param {string} text Old text. + * @return {!Array.>} Two element Array, containing the + * new text and an array of boolean values. + */ +diff_match_patch.prototype.patch_apply = function(patches, text) { + if (patches.length == 0) { + return [text, []]; + } + + // Deep copy the patches so that no changes are made to originals. + patches = this.patch_deepCopy(patches); + + var nullPadding = this.patch_addPadding(patches); + text = nullPadding + text + nullPadding; + + this.patch_splitMax(patches); + // delta keeps track of the offset between the expected and actual location + // of the previous patch. If there are patches expected at positions 10 and + // 20, but the first patch was found at 12, delta is 2 and the second patch + // has an effective expected position of 22. + var delta = 0; + var results = []; + for (var x = 0; x < patches.length; x++) { + var expected_loc = patches[x].start2 + delta; + var text1 = this.diff_text1(patches[x].diffs); + var start_loc; + var end_loc = -1; + if (text1.length > this.Match_MaxBits) { + // patch_splitMax will only provide an oversized pattern in the case of + // a monster delete. + start_loc = this.match_main(text, text1.substring(0, this.Match_MaxBits), + expected_loc); + if (start_loc != -1) { + end_loc = this.match_main(text, + text1.substring(text1.length - this.Match_MaxBits), + expected_loc + text1.length - this.Match_MaxBits); + if (end_loc == -1 || start_loc >= end_loc) { + // Can't find valid trailing context. Drop this patch. + start_loc = -1; + } + } + } else { + start_loc = this.match_main(text, text1, expected_loc); + } + if (start_loc == -1) { + // No match found. :( + results[x] = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= patches[x].length2 - patches[x].length1; + } else { + // Found a match. :) + results[x] = true; + delta = start_loc - expected_loc; + var text2; + if (end_loc == -1) { + text2 = text.substring(start_loc, start_loc + text1.length); + } else { + text2 = text.substring(start_loc, end_loc + this.Match_MaxBits); + } + if (text1 == text2) { + // Perfect match, just shove the replacement text in. + text = text.substring(0, start_loc) + + this.diff_text2(patches[x].diffs) + + text.substring(start_loc + text1.length); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + var diffs = this.diff_main(text1, text2, false); + if (text1.length > this.Match_MaxBits && + this.diff_levenshtein(diffs) / text1.length > + this.Patch_DeleteThreshold) { + // The end points match, but the content is unacceptably bad. + results[x] = false; + } else { + this.diff_cleanupSemanticLossless(diffs); + var index1 = 0; + var index2; + for (var y = 0; y < patches[x].diffs.length; y++) { + var mod = patches[x].diffs[y]; + if (mod[0] !== DIFF_EQUAL) { + index2 = this.diff_xIndex(diffs, index1); + } + if (mod[0] === DIFF_INSERT) { // Insertion + text = text.substring(0, start_loc + index2) + mod[1] + + text.substring(start_loc + index2); + } else if (mod[0] === DIFF_DELETE) { // Deletion + text = text.substring(0, start_loc + index2) + + text.substring(start_loc + this.diff_xIndex(diffs, + index1 + mod[1].length)); + } + if (mod[0] !== DIFF_DELETE) { + index1 += mod[1].length; + } + } + } + } + } + } + // Strip the padding off. + text = text.substring(nullPadding.length, text.length - nullPadding.length); + return [text, results]; +}; + + +/** + * Add some padding on text start and end so that edges can match something. + * Intended to be called only from within patch_apply. + * @param {!Array.} patches Array of Patch objects. + * @return {string} The padding string added to each side. + */ +diff_match_patch.prototype.patch_addPadding = function(patches) { + var paddingLength = this.Patch_Margin; + var nullPadding = ''; + for (var x = 1; x <= paddingLength; x++) { + nullPadding += String.fromCharCode(x); + } + + // Bump all the patches forward. + for (var x = 0; x < patches.length; x++) { + patches[x].start1 += paddingLength; + patches[x].start2 += paddingLength; + } + + // Add some padding on start of first diff. + var patch = patches[0]; + var diffs = patch.diffs; + if (diffs.length == 0 || diffs[0][0] != DIFF_EQUAL) { + // Add nullPadding equality. + diffs.unshift(new diff_match_patch.Diff(DIFF_EQUAL, nullPadding)); + patch.start1 -= paddingLength; // Should be 0. + patch.start2 -= paddingLength; // Should be 0. + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs[0][1].length) { + // Grow first equality. + var extraLength = paddingLength - diffs[0][1].length; + diffs[0][1] = nullPadding.substring(diffs[0][1].length) + diffs[0][1]; + patch.start1 -= extraLength; + patch.start2 -= extraLength; + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + // Add some padding on end of last diff. + patch = patches[patches.length - 1]; + diffs = patch.diffs; + if (diffs.length == 0 || diffs[diffs.length - 1][0] != DIFF_EQUAL) { + // Add nullPadding equality. + diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, nullPadding)); + patch.length1 += paddingLength; + patch.length2 += paddingLength; + } else if (paddingLength > diffs[diffs.length - 1][1].length) { + // Grow last equality. + var extraLength = paddingLength - diffs[diffs.length - 1][1].length; + diffs[diffs.length - 1][1] += nullPadding.substring(0, extraLength); + patch.length1 += extraLength; + patch.length2 += extraLength; + } + + return nullPadding; +}; + + +/** + * Look through the patches and break up any which are longer than the maximum + * limit of the match algorithm. + * Intended to be called only from within patch_apply. + * @param {!Array.} patches Array of Patch objects. + */ +diff_match_patch.prototype.patch_splitMax = function(patches) { + var patch_size = this.Match_MaxBits; + for (var x = 0; x < patches.length; x++) { + if (patches[x].length1 <= patch_size) { + continue; + } + var bigpatch = patches[x]; + // Remove the big old patch. + patches.splice(x--, 1); + var start1 = bigpatch.start1; + var start2 = bigpatch.start2; + var precontext = ''; + while (bigpatch.diffs.length !== 0) { + // Create one of several smaller patches. + var patch = new diff_match_patch.patch_obj(); + var empty = true; + patch.start1 = start1 - precontext.length; + patch.start2 = start2 - precontext.length; + if (precontext !== '') { + patch.length1 = patch.length2 = precontext.length; + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, precontext)); + } + while (bigpatch.diffs.length !== 0 && + patch.length1 < patch_size - this.Patch_Margin) { + var diff_type = bigpatch.diffs[0][0]; + var diff_text = bigpatch.diffs[0][1]; + if (diff_type === DIFF_INSERT) { + // Insertions are harmless. + patch.length2 += diff_text.length; + start2 += diff_text.length; + patch.diffs.push(bigpatch.diffs.shift()); + empty = false; + } else if (diff_type === DIFF_DELETE && patch.diffs.length == 1 && + patch.diffs[0][0] == DIFF_EQUAL && + diff_text.length > 2 * patch_size) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.length; + start1 += diff_text.length; + empty = false; + patch.diffs.push(new diff_match_patch.Diff(diff_type, diff_text)); + bigpatch.diffs.shift(); + } else { + // Deletion or equality. Only take as much as we can stomach. + diff_text = diff_text.substring(0, + patch_size - patch.length1 - this.Patch_Margin); + patch.length1 += diff_text.length; + start1 += diff_text.length; + if (diff_type === DIFF_EQUAL) { + patch.length2 += diff_text.length; + start2 += diff_text.length; + } else { + empty = false; + } + patch.diffs.push(new diff_match_patch.Diff(diff_type, diff_text)); + if (diff_text == bigpatch.diffs[0][1]) { + bigpatch.diffs.shift(); + } else { + bigpatch.diffs[0][1] = + bigpatch.diffs[0][1].substring(diff_text.length); + } + } + } + // Compute the head context for the next patch. + precontext = this.diff_text2(patch.diffs); + precontext = + precontext.substring(precontext.length - this.Patch_Margin); + // Append the end context for this patch. + var postcontext = this.diff_text1(bigpatch.diffs) + .substring(0, this.Patch_Margin); + if (postcontext !== '') { + patch.length1 += postcontext.length; + patch.length2 += postcontext.length; + if (patch.diffs.length !== 0 && + patch.diffs[patch.diffs.length - 1][0] === DIFF_EQUAL) { + patch.diffs[patch.diffs.length - 1][1] += postcontext; + } else { + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, postcontext)); + } + } + if (!empty) { + patches.splice(++x, 0, patch); + } + } + } +}; + + +/** + * Take a list of patches and return a textual representation. + * @param {!Array.} patches Array of Patch objects. + * @return {string} Text representation of patches. + */ +diff_match_patch.prototype.patch_toText = function(patches) { + var text = []; + for (var x = 0; x < patches.length; x++) { + text[x] = patches[x]; + } + return text.join(''); +}; + + +/** + * Parse a textual representation of patches and return a list of Patch objects. + * @param {string} textline Text representation of patches. + * @return {!Array.} Array of Patch objects. + * @throws {!Error} If invalid input. + */ +diff_match_patch.prototype.patch_fromText = function(textline) { + var patches = []; + if (!textline) { + return patches; + } + var text = textline.split('\n'); + var textPointer = 0; + var patchHeader = /^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$/; + while (textPointer < text.length) { + var m = text[textPointer].match(patchHeader); + if (!m) { + throw new Error('Invalid patch string: ' + text[textPointer]); + } + var patch = new diff_match_patch.patch_obj(); + patches.push(patch); + patch.start1 = parseInt(m[1], 10); + if (m[2] === '') { + patch.start1--; + patch.length1 = 1; + } else if (m[2] == '0') { + patch.length1 = 0; + } else { + patch.start1--; + patch.length1 = parseInt(m[2], 10); + } + + patch.start2 = parseInt(m[3], 10); + if (m[4] === '') { + patch.start2--; + patch.length2 = 1; + } else if (m[4] == '0') { + patch.length2 = 0; + } else { + patch.start2--; + patch.length2 = parseInt(m[4], 10); + } + textPointer++; + + while (textPointer < text.length) { + var sign = text[textPointer].charAt(0); + try { + var line = decodeURI(text[textPointer].substring(1)); + } catch (ex) { + // Malformed URI sequence. + throw new Error('Illegal escape in patch_fromText: ' + line); + } + if (sign == '-') { + // Deletion. + patch.diffs.push(new diff_match_patch.Diff(DIFF_DELETE, line)); + } else if (sign == '+') { + // Insertion. + patch.diffs.push(new diff_match_patch.Diff(DIFF_INSERT, line)); + } else if (sign == ' ') { + // Minor equality. + patch.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL, line)); + } else if (sign == '@') { + // Start of next patch. + break; + } else if (sign === '') { + // Blank line? Whatever. + } else { + // WTF? + throw new Error('Invalid patch mode "' + sign + '" in: ' + line); + } + textPointer++; + } + } + return patches; +}; + + +/** + * Class representing one patch operation. + * @constructor + */ +diff_match_patch.patch_obj = function() { + /** @type {!Array.} */ + this.diffs = []; + /** @type {?number} */ + this.start1 = null; + /** @type {?number} */ + this.start2 = null; + /** @type {number} */ + this.length1 = 0; + /** @type {number} */ + this.length2 = 0; +}; + + +/** + * Emulate GNU diff's format. + * Header: @@ -382,8 +481,9 @@ + * Indices are printed as 1-based, not 0-based. + * @return {string} The GNU diff string. + */ +diff_match_patch.patch_obj.prototype.toString = function() { + var coords1, coords2; + if (this.length1 === 0) { + coords1 = this.start1 + ',0'; + } else if (this.length1 == 1) { + coords1 = this.start1 + 1; + } else { + coords1 = (this.start1 + 1) + ',' + this.length1; + } + if (this.length2 === 0) { + coords2 = this.start2 + ',0'; + } else if (this.length2 == 1) { + coords2 = this.start2 + 1; + } else { + coords2 = (this.start2 + 1) + ',' + this.length2; + } + var text = ['@@ -' + coords1 + ' +' + coords2 + ' @@\n']; + var op; + // Escape the body of the patch with %xx notation. + for (var x = 0; x < this.diffs.length; x++) { + switch (this.diffs[x][0]) { + case DIFF_INSERT: + op = '+'; + break; + case DIFF_DELETE: + op = '-'; + break; + case DIFF_EQUAL: + op = ' '; + break; + } + text[x + 1] = op + encodeURI(this.diffs[x][1]) + '\n'; + } + return text.join('').replace(/%20/g, ' '); +}; + +// CLOSURE:begin_strip +// Lines below here will not be included in the Closure-compatible library. + +// Export these global variables so that they survive Google's JS compiler. +// In a browser, 'this' will be 'window'. +// Users of node.js should 'require' the uncompressed version since Google's +// JS compiler may break the following exports for non-browser environments. +/** @suppress {globalThis} */ +this['diff_match_patch'] = diff_match_patch; +/** @suppress {globalThis} */ +this['DIFF_DELETE'] = DIFF_DELETE; +/** @suppress {globalThis} */ +this['DIFF_INSERT'] = DIFF_INSERT; +/** @suppress {globalThis} */ +this['DIFF_EQUAL'] = DIFF_EQUAL; From 55ccf083c836043c7c242448762571494d41a75d Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:32:36 -0500 Subject: [PATCH 13/17] Add debug diff view --- background.js | 11 ++++++++++- options/options.html | 6 ++++++ options/options.js | 12 +++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/background.js b/background.js index b5667e6..58a5de5 100644 --- a/background.js +++ b/background.js @@ -33,6 +33,7 @@ let userTheme = 'auto'; let currentTheme = 'light'; let detectSystemTheme; let errorPending = false; +let showDebugTab = false; const ERROR_NOTIFICATION_ID = 'sortana-error'; function normalizeRules(rules) { @@ -262,12 +263,16 @@ async function processMessage(id) { try { const full = await messenger.messages.getFull(id); let text = buildEmailText(full); + const originalText = text; if (tokenReduction && maxTokens > 0) { const limit = Math.floor(maxTokens * 0.9); if (text.length > limit) { text = text.slice(0, limit); } } + if (showDebugTab) { + await storage.local.set({ lastFullText: originalText, lastPromptText: text }); + } let hdr; let currentTags = []; let alreadyRead = false; @@ -425,7 +430,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -440,6 +445,7 @@ async function clearCacheForMessages(idsInput) { maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; } errorPending = store.errorPending === true; + showDebugTab = store.showDebugTab === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { Object.assign(timingStats, savedStats.classifyStats); @@ -494,6 +500,9 @@ async function clearCacheForMessages(idsInput) { tokenReduction = changes.tokenReduction.newValue === true; logger.aiLog("tokenReduction updated from storage change", { debug: true }, tokenReduction); } + if (changes.showDebugTab) { + showDebugTab = changes.showDebugTab.newValue === true; + } if (changes.errorPending) { errorPending = changes.errorPending.newValue === true; updateActionIcon(); diff --git a/options/options.html b/options/options.html index d14ce99..3618a20 100644 --- a/options/options.html +++ b/options/options.html @@ -31,6 +31,10 @@ .tag { --bulma-tag-h: 318; } + #diff-display { + white-space: pre-wrap; + font-family: monospace; + } @@ -286,9 +290,11 @@ Debug 
    
+                
    
             
         
     
+    
     
 
 
diff --git a/options/options.js b/options/options.js
index ebd7497..3a09d37 100644
--- a/options/options.js
+++ b/options/options.js
@@ -21,7 +21,9 @@ document.addEventListener('DOMContentLoaded', async () => {
         'aiCache',
         'theme',
         'showDebugTab',
-        'lastPayload'
+        'lastPayload',
+        'lastFullText',
+        'lastPromptText'
     ]);
     const tabButtons = document.querySelectorAll('#main-tabs li');
     const tabs = document.querySelectorAll('.tab-content');
@@ -67,9 +69,17 @@ document.addEventListener('DOMContentLoaded', async () => {
 
     await applyTheme(themeSelect.value);
     const payloadDisplay = document.getElementById('payload-display');
+    const diffDisplay = document.getElementById('diff-display');
     if (defaults.lastPayload) {
         payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2);
     }
+    if (defaults.lastFullText && defaults.lastPromptText && diff_match_patch) {
+        const dmp = new diff_match_patch();
+        dmp.Diff_EditCost = 4;
+        const diffs = dmp.diff_main(defaults.lastFullText, defaults.lastPromptText);
+        dmp.diff_cleanupEfficiency(diffs);
+        diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
+    }
     themeSelect.addEventListener('change', async () => {
         markDirty();
         await applyTheme(themeSelect.value);

From 841a697c69a9204bcaf7c22c30be8fd5259837f5 Mon Sep 17 00:00:00 2001
From: Jordan Wages 
Date: Sat, 19 Jul 2025 19:46:44 -0500
Subject: [PATCH 14/17] Update debug tab with live refresh

---
 README.md          |  2 +-
 options/options.js | 37 +++++++++++++++++++++++++++++++++----
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 64fbf26..82649d0 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ message meets a specified criterion.
 - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page.
 - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service.
 - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service.
-- **Debug tab** – view the last request payload sent to the AI service.
+- **Debug tab** – view the last request payload and message diff with live updates.
 - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override.
 - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range.
 - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match.
diff --git a/options/options.js b/options/options.js
index 3a09d37..3402dcf 100644
--- a/options/options.js
+++ b/options/options.js
@@ -70,13 +70,18 @@ document.addEventListener('DOMContentLoaded', async () => {
     await applyTheme(themeSelect.value);
     const payloadDisplay = document.getElementById('payload-display');
     const diffDisplay = document.getElementById('diff-display');
-    if (defaults.lastPayload) {
-        payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2);
+
+    let lastFullText = defaults.lastFullText || '';
+    let lastPromptText = defaults.lastPromptText || '';
+    let lastPayload = defaults.lastPayload ? JSON.stringify(defaults.lastPayload, null, 2) : '';
+
+    if (lastPayload) {
+        payloadDisplay.textContent = lastPayload;
     }
-    if (defaults.lastFullText && defaults.lastPromptText && diff_match_patch) {
+    if (lastFullText && lastPromptText && diff_match_patch) {
         const dmp = new diff_match_patch();
         dmp.Diff_EditCost = 4;
-        const diffs = dmp.diff_main(defaults.lastFullText, defaults.lastPromptText);
+        const diffs = dmp.diff_main(lastFullText, lastPromptText);
         dmp.diff_cleanupEfficiency(diffs);
         diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
     }
@@ -729,6 +734,30 @@ document.addEventListener('DOMContentLoaded', async () => {
         } catch {
             cacheCountEl.textContent = '?';
         }
+
+        try {
+            if (debugTabToggle.checked) {
+                const latest = await storage.local.get(['lastPayload', 'lastFullText', 'lastPromptText']);
+                const payloadStr = latest.lastPayload ? JSON.stringify(latest.lastPayload, null, 2) : '';
+                if (payloadStr !== lastPayload) {
+                    lastPayload = payloadStr;
+                    payloadDisplay.textContent = payloadStr;
+                }
+                if (latest.lastFullText !== lastFullText || latest.lastPromptText !== lastPromptText) {
+                    lastFullText = latest.lastFullText || '';
+                    lastPromptText = latest.lastPromptText || '';
+                    if (lastFullText && lastPromptText && diff_match_patch) {
+                        const dmp = new diff_match_patch();
+                        dmp.Diff_EditCost = 4;
+                        const diffs = dmp.diff_main(lastFullText, lastPromptText);
+                        dmp.diff_cleanupEfficiency(diffs);
+                        diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
+                    } else {
+                        diffDisplay.innerHTML = '';
+                    }
+                }
+            }
+        } catch {}
     }
 
     refreshMaintenance();

From bcac4ad01709c85002e2f8123406bacffcf1d4cd Mon Sep 17 00:00:00 2001
From: Jordan Wages 
Date: Sat, 19 Jul 2025 21:58:37 -0500
Subject: [PATCH 15/17] Improve debug tab layout

---
 options/options.html | 15 +++++++++------
 options/options.js   | 22 ++++++++++++++++++++--
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/options/options.html b/options/options.html
index 3618a20..ddc5ee0 100644
--- a/options/options.html
+++ b/options/options.html
@@ -154,6 +154,11 @@
                              Aggressive token reduction
                         
                     
+                    
    
+                        
+                    
    
                     
    
                         
                         
    
@@ -220,11 +225,6 @@
                             
                         
    
                     
    
-                    
    
-                        
-                    
    
                 
             
 
@@ -290,7 +290,10 @@
                     Debug
                 
                 
    
-                
    
+                
             
         
     
diff --git a/options/options.js b/options/options.js
index 3402dcf..d881d6a 100644
--- a/options/options.js
+++ b/options/options.js
@@ -70,6 +70,7 @@ document.addEventListener('DOMContentLoaded', async () => {
     await applyTheme(themeSelect.value);
     const payloadDisplay = document.getElementById('payload-display');
     const diffDisplay = document.getElementById('diff-display');
+    const diffContainer = document.getElementById('diff-container');
 
     let lastFullText = defaults.lastFullText || '';
     let lastPromptText = defaults.lastPromptText || '';
@@ -83,7 +84,16 @@ document.addEventListener('DOMContentLoaded', async () => {
         dmp.Diff_EditCost = 4;
         const diffs = dmp.diff_main(lastFullText, lastPromptText);
         dmp.diff_cleanupEfficiency(diffs);
-        diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
+        const hasDiff = diffs.some(d => d[0] !== 0);
+        if (hasDiff) {
+            diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
+            diffContainer.classList.remove('is-hidden');
+        } else {
+            diffDisplay.innerHTML = '';
+            diffContainer.classList.add('is-hidden');
+        }
+    } else {
+        diffContainer.classList.add('is-hidden');
     }
     themeSelect.addEventListener('change', async () => {
         markDirty();
@@ -751,9 +761,17 @@ document.addEventListener('DOMContentLoaded', async () => {
                         dmp.Diff_EditCost = 4;
                         const diffs = dmp.diff_main(lastFullText, lastPromptText);
                         dmp.diff_cleanupEfficiency(diffs);
-                        diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
+                        const hasDiff = diffs.some(d => d[0] !== 0);
+                        if (hasDiff) {
+                            diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs);
+                            diffContainer.classList.remove('is-hidden');
+                        } else {
+                            diffDisplay.innerHTML = '';
+                            diffContainer.classList.add('is-hidden');
+                        }
                     } else {
                         diffDisplay.innerHTML = '';
+                        diffContainer.classList.add('is-hidden');
                     }
                 }
             }

From 9cad2674e3841fbc40d237f7233fbd6784f61760 Mon Sep 17 00:00:00 2001
From: Jordan Wages 
Date: Sat, 19 Jul 2025 22:44:04 -0500
Subject: [PATCH 16/17] Capture raw message text for debug

---
 README.md     |  2 +-
 background.js | 31 ++++++++++++++++++++++++++-----
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 82649d0..57a0285 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ message meets a specified criterion.
 - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page.
 - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service.
 - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service.
-- **Debug tab** – view the last request payload and message diff with live updates.
+- **Debug tab** – view the last request payload and a diff between the unaltered message text and the final prompt.
 - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override.
 - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range.
 - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match.
diff --git a/background.js b/background.js
index 58a5de5..fc585ff 100644
--- a/background.js
+++ b/background.js
@@ -210,17 +210,38 @@ function collectText(part, bodyParts, attachments) {
     }
 }
 
-function buildEmailText(full) {
+function collectRawText(part, bodyParts, attachments) {
+    if (part.parts && part.parts.length) {
+        for (const p of part.parts) collectRawText(p, bodyParts, attachments);
+        return;
+    }
+    const ct = (part.contentType || "text/plain").toLowerCase();
+    const cd = (part.headers?.["content-disposition"]?.[0] || "").toLowerCase();
+    const body = String(part.body || "");
+    if (cd.includes("attachment") || !ct.startsWith("text/")) {
+        const nameMatch = /filename\s*=\s*"?([^";]+)/i.exec(cd) || /name\s*=\s*"?([^";]+)/i.exec(part.headers?.["content-type"]?.[0] || "");
+        const name = nameMatch ? nameMatch[1] : "";
+        attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`);
+    } else if (ct.startsWith("text/html")) {
+        const doc = new DOMParser().parseFromString(body, 'text/html');
+        bodyParts.push(doc.body.textContent || "");
+    } else {
+        bodyParts.push(body);
+    }
+}
+
+function buildEmailText(full, applyTransforms = true) {
     const bodyParts = [];
     const attachments = [];
-    collectText(full, bodyParts, attachments);
+    const collect = applyTransforms ? collectText : collectRawText;
+    collect(full, bodyParts, attachments);
     const headers = Object.entries(full.headers || {})
         .map(([k, v]) => `${k}: ${v.join(' ')}`)
         .join('\n');
     const attachInfo = `Attachments: ${attachments.length}` +
         (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : "");
     let combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim();
-    if (tokenReduction) {
+    if (applyTransforms && tokenReduction) {
         const seen = new Set();
         combined = combined.split('\n').filter(l => {
             if (seen.has(l)) return false;
@@ -228,7 +249,7 @@ function buildEmailText(full) {
             return true;
         }).join('\n');
     }
-    return sanitizeString(combined);
+    return applyTransforms ? sanitizeString(combined) : combined;
 }
 
 function updateTimingStats(elapsed) {
@@ -262,8 +283,8 @@ async function processMessage(id) {
     updateActionIcon();
     try {
         const full = await messenger.messages.getFull(id);
+        const originalText = buildEmailText(full, false);
         let text = buildEmailText(full);
-        const originalText = text;
         if (tokenReduction && maxTokens > 0) {
             const limit = Math.floor(maxTokens * 0.9);
             if (text.length > limit) {

From c622c07c66a6936c4c8d3bb039b1f0bba4692384 Mon Sep 17 00:00:00 2001
From: Jordan Wages 
Date: Mon, 18 Aug 2025 19:39:35 -0500
Subject: [PATCH 17/17] Supporting v140+

After testing in Betterbird v140, updating manifest.
---
 manifest.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/manifest.json b/manifest.json
index a3c9f7c..e7cb9d8 100644
--- a/manifest.json
+++ b/manifest.json
@@ -1,13 +1,13 @@
 {
   "manifest_version": 2,
   "name": "Sortana",
-  "version": "2.1.2",
+  "version": "2.2.0",
   "default_locale": "en-US",
   "applications": {
     "gecko": {
       "id": "ai-filter@jordanwages",
       "strict_min_version": "128.0",
-      "strict_max_version": "139.*"
+      "strict_max_version": "140.*"
     }
   },
   "icons": {