From db7113b03d81faed2da12ddf3076cb0a6408f4f4 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 01:38:11 -0500 Subject: [PATCH 001/124] Create brain.png --- resources/img/brain.png | Bin 0 -> 13993 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 resources/img/brain.png diff --git a/resources/img/brain.png b/resources/img/brain.png new file mode 100644 index 0000000000000000000000000000000000000000..296bb646386cd6cbcbc028e261e84bc130a313a6 GIT binary patch literal 13993 zcmeIZhgVbS+bH_(B!nOdM2b=b!k{<~iYSC$0$6YqMNv@{3F-(2g*b>bflV>k5FBUJ z!60!|1O-76r4vNK85K|?7Fw_XA|*<%xzA>P-(BB5-yd+-S?6Re7kj_u>Fs^q?FojALU#Z(-9a{ zJiNwzrQ#Q=E>%xMV-iyYkZ8>c=gr~ods=pEn&Ku1=iDOzrCfX#e;3f1m`sOCM0sy20_$!TXP6>#Ls@JowFr zx3(N(fi14AzHlY^COfkcF%jY-^Q_rB{ks_mLr7po)`<(vY@L~4nv1g8LsT53N8c?l zFZ_2HcYemtY?SCC5<+Ijwd*8l-ee8{b*7WkB0U2U4T-|`WUqM`QN_<}mFU1S8gDUs zjrCHkI?1J(rOwIMjg|Zr$Z2Vw{;-^Ntyscp7jikCA>DK!BGKYvnvrleJ%G199b+Lp zmFl@C(L5e=Z^<@C6vTYV<(=fOG81LnunF2+($=}pu2ZC1uy51>*&Mg4*@h3t#CT?h zM4Ldq--7LGtvkPM^?$%+?ejEGEO7pieyv0D=N4ke-h%Ae4~tn})^7?6pCB-_9N;DL zO)thQ&9*6v1sG(R3VJ+Hvpkp*r&0CC!{40oOtVU-x+~c0)i}@t<~e$LG_Z0-UY$H zb;A>oit$6<`|`{hNm?|3!fRHlCz{XRTHR*@#?@m2fD+DRZc|U}aL5X3i6B7v0w}7U zh0CM#6NBENiZ_>r?yJNoD8Q|OW;2)fs^V&mHs%Ik#EGYz($%0PgHuFf7Iqwt2YM9Z)+(-brpU}o{RwQje2bk;Hw&pjH9t6_7% zMfQq1fvx949^DWTtf@O$si`}j5hxn;~x`eEay4|L-zzt%M?RN&UP@Zeas8GU(-fZ3xo8g&@HJn|m zm+<@#Mv9?IwNv++WbhqI!--%dvMIW|QRa@+_FwBws=wBVx}N@W#`${uDR4XmoX#VjMT9>2BzEUUA@$7a_ z#zv_~Ab+kyYNK#*8J(6pv+naipK#(bW-3k^__(z1QtYdZx02VW@m_3^$J&LCl)VG?0qV;Q1 zsViuBB&B&7sKQKo%tGG<`Mc@uW<9!028dF%mwbRIwxhy}tJde-EOt-0&k5j9UX5P$ry5S89CIVk|CAG>nYfXj%zM*gEpDI)F9Qsy4tZQC* zVP17(Tk#cM?@jmSY!o;~#cL%Afj*Lex9>=@OrIko6Z??aG3==<&oKYZa72T*o!#c! zdxrkLHnPrvtOv^#B?-KANsm>mqtiD`tc$JlYgj(1%j+$VHn^8r$Nb}~cl2K=*tVxu z$UaTq{cEP6ZOVo&N7pKhKu8JasveK8l-w0CkTW3)@!s&lGu46K=cqV~VAx~%z56@r z`vM~mAGBMG`pkkEu*m4cod@@M^N?j!Q&}(osdj4I1|nw>KG$wwi0U-(3Kb5Lo;Ay2 zQ4oGVSQw@Putsa&D`TkPE7*=bjp_kqj;NE&MBz00Wg6DA2rX%|3D#qQTy@vyFLXg@ zkO+hqDxB2~5$aL|t(iuVjbS`stTCb=VR(BR_e8lG$vO~4u7KB5Eu45J=Q&B_1#zXQ zf$@4QLS{VVB0 zXOzFHTU0`GlndhSFx_(DZhOYJp(yv3tz*yX=AEm%6IpZVlMo{z)jt#!LKjo`tq}c2 z{@G4G(&2^dUG&v{;EA4YAN<1$KiIc1Y>~Kf=V>ZLg^i)n`yC9vMWip=RCn9E)`s5- ztSZ)gj2k~g$An8MsJX z@XoJqVuJjYG2EOlsn{0ihev|t>N8X+-7!gN@*swAk#I|=-{;HQLJV;p&EbH^EdDBCO3*~GZ-?-)r2Mezd;cgE8rnH*8ZK1# zf8^L?g5iZ>bKeI>sm$xxt3;RcS4@qa%(mm_$q$a`9*C_Fn)K_LovS3XJ}tXD>zc~R zo`Q@&h9yq>shhu+mJIFc^B^pgG*s*4dgSv|kIbJE7Pghit;fJzLM3{`-$Z8OzBs#` zP<$X+YC@zD_xRQ%DP0Ncv0|a)r42qkSp^ZD@1z-H5#sdQ{`cCzFXq~HD^J|8tQwmQ zMFs4-dV4gcUu}3pGRXynEr1A;Xc4JJ$j5rbXY~$+TI)6o> z?$p(CDny|WEnlaOBdJjBYiRmhvJ0pO7ZgG-+wUhYXbd}zXFv-%Hk*5X|`$xkpa~DO@z$?SW?8jQu*dVi2 zEh?xI!|G3vU-?(@pmnxY;XdEB zz3&q1MV7)QNl0+YKwm_ZO8F=iSC)8WM~Ex>BJ+I4J!4{JWq6%DhWew(qZ^HFr8 zIgMZ0zWz}R*z?Yww7nA_j^@9T2Gv*DI0qB4?2|r)F4t&kD<6K+N-k0-sA}YvOcwa^Fz*4Y~BF zbV`A+*rS%AHL?GT3F1(R%-8?-F?_cX=nJkTwI({FdH(DwgJk`psdj>x-Ce zW|Ky=w@a-(M&@5$c8jPP3~{Hc-iHKF6|ju-xc=#2gqr*IDverm){U(lx~bwSh`o4d zj(Gd`Rp{_XU35CS%tU3bAl5@J^}!&`tO^U?vv;)C5o^^X#6DkOLXX;XM^(EtKr(xB%<37tK$v_@w;{h|P$%H7?-&!^O ziBf5w_1&|}C-*GH`ez}{^u^d4x~6*C**CQ}9^@*{ua2E6T&H{ThVaec0AAI$ouKB* zUmi5>X#brU19_OmZLI(^0xoda?j)*;miu3{9UPm3v?gzq8=#%FZ5W;;7YG6$wmm7^ zt$OQr)2B+m9p{2_v*apD-QW$P?WX#qx1|w`KFO+PZeTLM&3Y+%H#rMkVqZ1fgUBD6Oa-wMLFwrZQQ3_W+F{6f{CB=G zzRq+((z8VTX}LmEo9?Ta+$VoR1Qd@l#j@D#1$O0sdMWprxao{z7o9&=`;iZKVpZY` zg`I1v_k72Hvz(mxIX`5tYI@iZ$@=$zlNJYty=hWkG2dWESm)#7LyfN?L2wGL`4!YI zN_KzfXl!+bg#4?=U_w>ekHp7wYPnfx~rD``vQJ)X>Q7asA>p;iAr+>2V0zZ_@Xe$jn~kt*Ema!zZ4<+r4V) zrj5107Za}xUc#>3xh2G;lGh(NUDQ@qZ=C>^<1Szing#AdWlzJcARVPK0y~`>pP{8XB(hRIlo$+KbO?Z zPdZ$(IuXRm`FN|HiW-_=&VHM|+YpACew?FPMwg^Md zzDAVwvk=~*zZinH=*A5HRHtn%)BAR&EJvvtBw)TMf^RiSMv_iN71`6j)x>Ro4{cCQ zjEl`!9r3w@Hw#IJcl{Fu8V=B-LRZF!vTcg2m?h3b=okIZ*W^Cvq7G@B0yKNHYA`S@ zI1in9u$<*)uI zec@#w&S`(@>J z&JSe|M`W_+-Fo9i_RsSO>t3&y#js8ZMx3*?>(ruKy?yVliun+`lU(>~Zn~bCEPbMr z?}ffhF2Fsq?nHp;ncutoU{zc0xloGPWRw9fTdN{VL2Zrmm9B(4<5vcM?6BZ?c0b-Q z+}(f9VN+n=FO%N-IiGcrG>+m=J??O-LX+Uvm1>>AY0%c7%)9!2LvybUF#XKv&i9Sq z|F9117Lu~Vjz_^k0|~VGL1=_#rFI+)$bzIF5O~5|xo&51{U_6nr7Q&Qt0y&W|2h5; zl>14QQh9~qdmY>jVOVr#b&27_1=y<9=Z{(%K-%)$UR$s`vsYv5^e=9tskOVSa^q8XDteb zxQzt%fxDMj^guOH3bhJKDJ;x4CaN<%g2T(~d`lfiert;f7>tY`IDYg<;HPKgK1~9w zi2l&AfQk4^OS~kiu==GzA92WWKM4W4oHVn&K8cBhAkAFIWo( z#*3q(UWL$^P*O0&>)I^N19jY6+gwWY&*iV=s}2m%LOs)dcq!*>}gpfwq3Xu zg&K66V7#2D-<*vtm*x3Ps63Z_{rKvVZQP!IpM>S&`$x%W+xy8aqia&H+^%JLETH7? z$nQwMc*Lu812_q@?bCK8Lp)Q@dLffZ;sygFIFqx7_7IgRfavUFM8n6B8JpfU>K}Fi zm_`jnWG@QVrv`iRnzibxiOmLmOn5;q_;aOH4pV9h2T#O;bCwbMMYtG4uP=?LkcWY! zrs4#F7&RuM*Y&LnMxdlc0`^awN~Sm{u9PDBsgi_#53I`}@yKrFYbhFiM}c8nz~o;b z^bT0p6|X(2OlSxP8UNQ{(OLx90-CY+%?n}}|Lg33ZQ&qQM-VZwlr=*b@B#JGe*oV* zJ02Jpqo?-YXB*1M#4M~Sas}o*daFiCgGF(Q4eB5RTrbUpaUQffMOa9DJh~-1p-AAS zVX%SNmq=)Igxbo5L4D5GlzsY$L*VAX9E$)8WVrMz2hV099Ip~5EG1^}2ZZEYkVPbF z^@io5W$nK)IRsc@2aSYA5=yav-n~kqT55r{m=|Zu1qR_CyyP`JX){qw>(x68o;soz7yoZvWW*3zQd#al>Q%Q z$VMysHNxiLAn^4^N3!Y}LFY%LW##ev+|~#;pX>XL=ms4TaY$aMS+OiCKkI@Ev4G@$kpPW)IE* zdAO_S^MVVcOwc00+!eBCUpq&L^Q z`G`E1gomk)ra|c`;M7?Bp|h6TE$xnmDx*P$y+PZID44XD!d4+6ez6fn9H9t)4WK1D zyxh5IHvG#+kMyum74+iH1Xu?8n`?ak$;hZiiofG(Hiz8t$DA zkh8&H9ImHud?4(}nBGxnL}NJipH4NZAsl=QlYG=k(#tPbvxve%bmeQ&++nh8pO&CO zIS#6hFE4)&m-Q&)c0U{VRZbjTUj^L?aY7JWj0G>HA!&f#Z`)Oct0>~BYbhfCgSVi_ z!84!7VBFCW>R2-i;gY1Lgv{Ahj`3-Nh@svcghg5WfHoR4oT;`11aWom>}vh@b{NkCU${zws6uC>U*{9&ms*A8dhBpPy=}U6J=z)YnnY+90DdO$Ea#3ykwuNi)q7=f@4&?Fd?=UfSUzQQ9NjfCH-|MoQzj+k|t=imWt@|aiTR!m~*vUqj3%e5(y?Yqg!8sBgQYY(9W-2X3ZPq zgrlNEi5~>9Ip!d}k0kVQbfFwf2kv@CZ#1V(oLAWb#h%{g;uQ{`q!oy*g2NPjrKqmB zv~<9gEHZ{@$C-$G;z1GY_?j>T)m`{H?x-wS)1$v@E&+$GpGbGb+cUa@|JcF-``I8) zN8PP5RTgN&mP?B-3GH{kXFN<9{|*&M?xh{7WwuyoinBVbrC8j}m z;=)s$UR6IGC~uF|C#o#5sAIr;&$g{2yvkN_kQ-&ws3wdJKksbRxVgyf_Jm59wBPCD z^{-!J6v5bQ#3Eygho`8cya^eNX(G-mo?gy^p3VK~fvp?{$c)ZwBnQ$G6k}1xcN3&F zCx3Wu&Z&&!*$dsWLc{cc#0GJEWPCe2Xw-kVdsK>aE)d8S6>jCPjJ|aU^nu4cx7}_O zm{?W0z-fv|%~Z>r3x!our?UGDLd3_fDUrx(WqAkt^`$C)q*x`E)UU*^#p3x?;E(%N z(?bKP1ditTb_19GRh@{H3_Iy!@uvrLfJZG1*OEai((-z{EVFO( zQpPr?#VHCE(9Lg<@{M#PL>a@#i@ZBax|pEWMQ3wzUS#~IkbhtQ60jNhk)%$9ra$*1 zw8}F3_4F=8x0lXC`4E;lsjta*R%sSe>M)=NJzqAgS!Yj%mp3vKICL{YDcx`#22G9( zS?NceeLV}ZVhIP(@scpRd-9#mjVq4KI`9~*e0EA+aA?3~PD^#~nV1dfP!MqgT-8s# zk2@QjQew4uU`of<2&fawR1dnMp&)7JclMDW4kFUe`$WWRk=FOH#M(E00Of=~b1tCx zN$>6;=WP*Cbh?_Fc2c!zPTtuO>Vy0s(KTI!o`O;%kE-LyFJVf5Kv73YL2&8&JMR%*H0u2`MSEC(D=`&WwpEOo1|6qw!4%cT~~e zrpxEs{qQ~I`ys15?lruxui+4^!+zT_PkXxDr18<8V9o+H)jOeL8Kdr7m(}Ca-O)l- zs91h1sB?5m`#~#Oz>v3U1b)r|g_2LMe%o0iQ>~MAjXb*aL2nKP2HnrB-gZ?`h0)Cf z+j26(QeBHu`HLcR>|JyiLvuDQy!GbQ*sn(zx3f$_AA5sO>SvuYxSXAmvih|RN$Asm zA~ZLqLbxob{a2TQ_A4iYzguYVOs?;OBSvt%j#qFuVdIpYb!SuZlRO=iP>!BpSz&IU z#9$5RDF5&3SH0i9ZR2&|2kUa$+oE>&mZuN4H%Pw^aNa3P+}#+EX#^@mIr^^`K(Y3% zG<9|ViCT}`tOanzxFj^k^7SGbF?>3yFF9$$!7LLPUOaSbJ<(_pAM0PPe;@937}ljE zB-~PXDs4+8tZpwoUbQ_mNC9bGljMwR5{1It@>5#E7s7#`;BNHVuGa-D@lPP@1^sGN ze{-pD7BX9$?HQVEI~U<=TNUVad&OfKh|AfOK_f|y-j^@N!V@{^fv3f?4KHJw27LW@ z8T1khx~oA(;Pq#2Lr0QV*>co;w_)fb8v=KBX1YCkFOsVMAbsJ3r{KX>fm` z^Q<2ehDy#%t3z`bx4e*lN{f+Thgl5O*>pdNj)R4d^2;n4aw+3b-JiQQ@Mb{=i)z8w z`l*?WJ2QqpCqwK?lSiBGnY&zvTQ}Z?|N>-kK`CzARKV#>rvueTbWj)`jA`a=_ z?8}(q?|Q9PmtkHzV?FbrhkCHCuprrr29;F&v{{xCiGSPh%_A#uaps=qH+MicF1rE! zbzbN%W**aNR0IokO;FaQD?y&p9-@OCEOT9>=Q7`aZe;qqkh*&z&n3G&4;sNeY~ntq zaJ%@=Rq0f-j~4pHyYBwnC>i1p}B2CFz0qD#fycwFAkU zAh4ds{M9Tv>XsikY${{`4QnvrN;4@w5@gdxfjkt*wsyI^uZoK0}c>IEg2jVkrZO z&rkhw-6_Qv?tWoZ!dEVxsXq}8uXx|&pn=e?lFEV?&Q%66Mr=iwz7MU#wI@Hva$XN~ zHm!lo3-btRU%P_VeS#~5c|X7aSCRCK*x?v4N9;j|JD1AICBoH&6o?<}`f1>tBh;QO zc;ot4tXH0Vl!wi{45nf@F-UP*N#?)<63amxkO0u>d8RYaPD^k2<56@BS$te*jKZHH ze@zUg7V-e?9$xEFsWPlii%t9*yeB07y~5y?J_jJ3s+=*GkS6Z;wo@>_qL7fjnAtOs;H{wZo%lYBbM-{@ zeU%vj(#spkFS}p8k^7l;>VmZ#Ubb%ATqM>d5Z!z0^_9Gc@|qAd{Q(}LOqVVtuK>7l z$^>|=#QY15%9(Gj`UdTLILU!dN_LJcZ6!P)9#hmSA=L3@P<_5CwHr(Uut6Df%b>Xpnn@s} zxU>JLN@qvi3=&V#i;RGzuqibU)#I6>9p$5o$OJ+rDgWc;o9daj=9t=q;H93Rcp5uR zSbk$i@2zo!9CU$RnphUSWiFv>asHEqKM9=P9M$DM4Q8&}wL|C|wp?1bR~1}k%JlJ> z=S_qYDMx3H42{0i*R@2VTDvm@k^|FX8p#4YV_tn8#H~lhl!emClBBr5!{T;i&6wgJho62N(?aX(S@i9!wL6ccGDDI&K_q zWEMX7oF3?f!mc93Kg9bP3)2vD#-Sl7sxlq3ju6p38&wG5B+{)a6ObS7JV)3P-M6ij zRHP0n~K|I9b6f@+HGT&rQsL34b!|qa24^hp*K~6naT-AF6)_4R5BkR`KDKr~ogA z`+rB}RA8b$SqeDN{D{N7cj^!L=A<|i7Z0LEa9eS=R%xQ=(h`9U8q5HSF628O7+5N97*b_>_>DMaF`$rne6!=)5`m+P2^^=X#-gAp5qdar9kaa)O}> z>PG``HsL;*)%I0oqIcnRQ4|KE3s}*VeX+(%FP|~pr`Eg$9jFC8I$8FmiJ$o${Osft zu#g^%o<%B_^+PZIg-5`}C(wyKFyDeAtx$mN8uO^^Oddp=%aeKtH)INCORk(pQ$`Rz z3f4BSR0$Z((e*edQvr_FC=f0o9x}wCtMXffWDakZ+ZWB;crYVKpMF$J<9iUaX(Y}o ze_R1p;vEJEO^Fba3f8tgma~0dxH`PJerT(&bO|74fk22wEOf@>+d{{fSqh^50z!k% zxS>+OTTW>nf_P=6h&@CXZGP9*!kUX=M+Kd1{}!N5*xq6xWNk;2@x}GFiK|Jsw#s^9&vGeEpy z=A&K&mjXuCNo$oBfowi+iO#WTT$iWFCISJXyJe9pqzexawkWWa2%M@b>ysQPz&g@m zVa*v&$Bz64AzZVKB(9jq=`$hN)k@88b2J}Tfkn8n?nF-kdGGh!`5-05XL#aNAyp?x z>~jJDq&N=0_I^8)L6yg{bp#9HMzUfJLY(aOC-~lmLQ9?CTc~dph$GSBbsBOPvLEik zScJcG+j2C;qSE}A(Oo;-W~h9-G;mHc-n5W^Kg*p*HtVYTqvqB^a$d>Z4br1_oj|+` z)l&7Yw;CmD$poeDenB$mE`UE^eap}g*D2T_wYPgy*4Yn%o|FGk|E(fBC__l*=WRUp ziOL;KSbW{fTj2q~(pM}uzd31+CZQs??K^dMEH*LSY(eIgUeX7n71e8*$*mil>y$6l1j3y(# zO1cvUsegWy*4>N~UtbkBPK{HoW9#%_p=!lGaYO9T^nyIETJMqG`TRbi6+!fLSB==I z@^Dkk_DaX0zW!j;`P>HlM5J__H>am|IBJjk1D`ib8V7F44_02Lv}$+4_kQ?3?oK1W z=P5o>Ut9LXz8d~JHU2?su?y^41{V+3Y*UI^zBD-u?jPV>^2445LUo?p_LOR^?QZ-*QK8Ul10%CZj`;suBrOT4lg#M$4Dgr-)krs_5a*3p3a=M)_R zEZ>neF0_W+f-L3)NF}2UX-taN5q_+V@}JWM@NwG?$samiO9l3HxgfTq;0w}WSnzrF zMXQeLikY7rEJuyN20z9buTNHdeIj-d3eNY2+<{Xm?m`{ao0`t3)u#~b2R)|&6Z)IY zvnPJ>_}+SxsAU4yH9X4xR#fusAS%1V(*h~$XjuHpv(GwmR2PMvDOm|{1PaLn97AW+ zr(Jjw;3O;S*s%5CzlfmbCEaiyLOR+rdoHVuJWc)VxbXs*Z$pFVglFfS&;>N{)D;!T zbfPL}qZ@Uuyr+2z(mIbQVo49N#F^dEgo=zL^9b%8f%^!z^f5~WUjZ8eSL%t>_34VQ zF9nlD6_Po`l9kkDl1PshC6b3H4=N6nHvhS9xBa!@M3iO3P6X=Q&t)qC}$mLOjI7CKmBv_%C>3 zQ?(NThDpJz{@sL$iR=drXH08yW|Ir_&soHXGZ-7tMBx5+6MQ9AW$-U<=+FI&n=1qR zATT)t6*F^`|C*z4WzZ4_x-hfJ8M^*Eiht>HhR=4666Grpq%}H&{s1?HBTj`V^M8N; tdr3enf1lt=s1iBLeWR-X{~ Date: Fri, 27 Jun 2025 01:39:02 -0500 Subject: [PATCH 002/124] Add reasoning cache and viewer --- background.js | 33 +++++++++++++- modules/AiClassifier.js | 92 ++++++++++++++++++++++++++++++++++++--- reasoning.html | 17 ++++++++ reasoning.js | 27 ++++++++++++ resources/img/brain.png | 44 +++++++++++++++++++ resources/reasonButton.js | 34 +++++++++++++++ 6 files changed, 240 insertions(+), 7 deletions(-) create mode 100644 reasoning.html create mode 100644 reasoning.js create mode 100644 resources/img/brain.png create mode 100644 resources/reasonButton.js diff --git a/background.js b/background.js index de6b595..17154b0 100644 --- a/background.js +++ b/background.js @@ -231,7 +231,8 @@ async function clearCacheForMessages(idsInput) { if (browser.messageDisplayScripts?.registerScripts) { try { await browser.messageDisplayScripts.registerScripts([ - { js: [browser.runtime.getURL("resources/clearCacheButton.js")] } + { js: [browser.runtime.getURL("resources/clearCacheButton.js")] }, + { js: [browser.runtime.getURL("resources/reasonButton.js")] } ]); } catch (e) { logger.aiLog("failed to register message display script", { level: 'warn' }, e); @@ -313,6 +314,36 @@ async function clearCacheForMessages(idsInput) { } catch (e) { logger.aiLog("failed to clear cache from message script", { level: 'error' }, e); } + } else if (msg?.type === "sortana:getReasons") { + try { + const id = msg.id; + const hdr = await messenger.messages.get(id); + const subject = hdr?.subject || ""; + if (!aiRules.length) { + const { aiRules: stored } = await storage.local.get("aiRules"); + aiRules = Array.isArray(stored) ? stored.map(r => { + if (r.actions) return r; + const actions = []; + if (r.tag) actions.push({ type: 'tag', tagKey: r.tag }); + if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); + const rule = { criterion: r.criterion, actions }; + if (r.stopProcessing) rule.stopProcessing = true; + return rule; + }) : []; + } + const reasons = []; + for (const rule of aiRules) { + const key = await sha256Hex(`${id}|${rule.criterion}`); + const reason = AiClassifier.getReason(key); + if (reason) { + reasons.push({ criterion: rule.criterion, reason }); + } + } + return { subject, reasons }; + } catch (e) { + logger.aiLog("failed to collect reasons", { level: 'error' }, e); + return { subject: '', reasons: [] }; + } } else { logger.aiLog("Unknown message type, ignoring", {level: 'warn'}, msg?.type); } diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index cec4a31..4180973 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -49,6 +49,8 @@ let gAiParams = { let gCache = new Map(); let gCacheLoaded = false; +let gReasonCache = new Map(); +let gReasonCacheLoaded = false; async function loadCache() { if (gCacheLoaded) { @@ -94,6 +96,50 @@ async function saveCache(updatedKey, updatedValue) { } } +async function loadReasonCache() { + if (gReasonCacheLoaded) { + return; + } + aiLog(`[AiClassifier] Loading reason cache`, {debug: true}); + try { + const { aiReasonCache } = await storage.local.get("aiReasonCache"); + if (aiReasonCache) { + for (let [k, v] of Object.entries(aiReasonCache)) { + aiLog(`[AiClassifier] ⮡ Loaded reason '${k}'`, {debug: true}); + gReasonCache.set(k, v); + } + aiLog(`[AiClassifier] Loaded ${gReasonCache.size} reason entries`, {debug: true}); + } else { + aiLog(`[AiClassifier] Reason cache is empty`, {debug: true}); + } + } catch (e) { + aiLog(`Failed to load reason cache`, {level: 'error'}, e); + } + gReasonCacheLoaded = true; +} + +function loadReasonCacheSync() { + if (!gReasonCacheLoaded) { + if (!Services?.tm?.spinEventLoopUntil) { + throw new Error("loadReasonCacheSync requires Services"); + } + let done = false; + loadReasonCache().finally(() => { done = true; }); + Services.tm.spinEventLoopUntil(() => done); + } +} + +async function saveReasonCache(updatedKey, updatedValue) { + if (typeof updatedKey !== "undefined") { + aiLog(`[AiClassifier] ⮡ Persisting reason '${updatedKey}'`, {debug: true}); + } + try { + await storage.local.set({ aiReasonCache: Object.fromEntries(gReasonCache) }); + } catch (e) { + aiLog(`Failed to save reason cache`, {level: 'error'}, e); + } +} + async function loadTemplate(name) { try { const url = typeof browser !== "undefined" && browser.runtime?.getURL @@ -185,6 +231,17 @@ function getCachedResult(cacheKey) { return null; } +function getReason(cacheKey) { + if (!gReasonCacheLoaded) { + if (Services?.tm?.spinEventLoopUntil) { + loadReasonCacheSync(); + } else { + return null; + } + } + return cacheKey ? gReasonCache.get(cacheKey) || null : null; +} + function buildPayload(text, criterion) { let payloadObj = Object.assign({ prompt: buildPrompt(text, criterion) @@ -199,7 +256,8 @@ function parseMatch(result) { const cleanedText = rawText.replace(/[\s\S]*?<\/think>/gi, "").trim(); aiLog('[AiClassifier] ⮡ Cleaned Response Text:', {debug: true}, cleanedText); const obj = JSON.parse(cleanedText); - return obj.matched === true || obj.match === true; + const matched = obj.matched === true || obj.match === true; + return { matched, reason: thinkText }; } function cacheResult(cacheKey, matched) { @@ -210,6 +268,14 @@ function cacheResult(cacheKey, matched) { } } +function cacheReason(cacheKey, reason) { + if (cacheKey) { + aiLog(`[AiClassifier] Caching reason '${cacheKey}'`, {debug: true}); + gReasonCache.set(cacheKey, reason); + saveReasonCache(cacheKey, reason); + } +} + async function removeCacheEntries(keys = []) { if (!Array.isArray(keys)) { keys = [keys]; @@ -223,9 +289,14 @@ async function removeCacheEntries(keys = []) { removed = true; aiLog(`[AiClassifier] Removed cache entry '${key}'`, {debug: true}); } + if (gReasonCache.delete(key)) { + removed = true; + aiLog(`[AiClassifier] Removed reason entry '${key}'`, {debug: true}); + } } if (removed) { await saveCache(); + await saveReasonCache(); } } @@ -233,6 +304,9 @@ function classifyTextSync(text, criterion, cacheKey = null) { if (!Services?.tm?.spinEventLoopUntil) { throw new Error("classifyTextSync requires Services"); } + if (!gReasonCacheLoaded) { + loadReasonCacheSync(); + } const cached = getCachedResult(cacheKey); if (cached !== null) { return cached; @@ -255,7 +329,9 @@ function classifyTextSync(text, criterion, cacheKey = null) { const json = await response.json(); aiLog(`[AiClassifier] Received response:`, {debug: true}, json); result = parseMatch(json); - cacheResult(cacheKey, result); + cacheResult(cacheKey, result.matched); + cacheReason(cacheKey, result.reason); + result = result.matched; } else { aiLog(`HTTP status ${response.status}`, {level: 'warn'}); result = false; @@ -275,6 +351,9 @@ async function classifyText(text, criterion, cacheKey = null) { if (!gCacheLoaded) { await loadCache(); } + if (!gReasonCacheLoaded) { + await loadReasonCache(); + } const cached = getCachedResult(cacheKey); if (cached !== null) { return cached; @@ -298,13 +377,14 @@ async function classifyText(text, criterion, cacheKey = null) { const result = await response.json(); aiLog(`[AiClassifier] Received response:`, {debug: true}, result); - const matched = parseMatch(result); - cacheResult(cacheKey, matched); - return matched; + const parsed = parseMatch(result); + cacheResult(cacheKey, parsed.matched); + cacheReason(cacheKey, parsed.reason); + return parsed.matched; } catch (e) { aiLog(`HTTP request failed`, {level: 'error'}, e); return false; } } -export { classifyText, classifyTextSync, setConfig, removeCacheEntries }; +export { classifyText, classifyTextSync, setConfig, removeCacheEntries, getReason }; diff --git a/reasoning.html b/reasoning.html new file mode 100644 index 0000000..bb1577d --- /dev/null +++ b/reasoning.html @@ -0,0 +1,17 @@ + + + + + AI Reasoning + + + +
+
+

+
+
+
+ + + diff --git a/reasoning.js b/reasoning.js new file mode 100644 index 0000000..eb0b9da --- /dev/null +++ b/reasoning.js @@ -0,0 +1,27 @@ +document.addEventListener('DOMContentLoaded', async () => { + const params = new URLSearchParams(location.search); + const id = parseInt(params.get('mid'), 10); + if (!id) return; + try { + const { subject, reasons } = await browser.runtime.sendMessage({ type: 'sortana:getReasons', id }); + document.getElementById('subject').textContent = subject; + const container = document.getElementById('rules'); + for (const r of reasons) { + const article = document.createElement('article'); + article.className = 'message mb-4'; + const header = document.createElement('div'); + header.className = 'message-header'; + header.innerHTML = `

${r.criterion}

`; + const body = document.createElement('div'); + body.className = 'message-body'; + const pre = document.createElement('pre'); + pre.textContent = r.reason; + body.appendChild(pre); + article.appendChild(header); + article.appendChild(body); + container.appendChild(article); + } + } catch (e) { + console.error('failed to load reasons', e); + } +}); diff --git a/resources/img/brain.png b/resources/img/brain.png new file mode 100644 index 0000000..7d9e7d6 --- /dev/null +++ b/resources/img/brain.png @@ -0,0 +1,44 @@ + + + + +Object not found! + + + + + +

Object not found!

+

+ + + The requested URL was not found on this server. + + + + If you entered the URL manually please check your + spelling and try again. + + + +

+

+If you think this is a server error, please contact +the webmaster. + +

+ +

Error 404

+
+ openmoji.org
+ Apache +
+ + + diff --git a/resources/reasonButton.js b/resources/reasonButton.js new file mode 100644 index 0000000..0206ab9 --- /dev/null +++ b/resources/reasonButton.js @@ -0,0 +1,34 @@ +(function() { + function addButton() { + const toolbar = document.querySelector("#header-view-toolbar") || + document.querySelector("#mail-toolbox toolbar"); + if (!toolbar || document.getElementById('sortana-reason-button')) return; + const button = document.createXULElement ? + document.createXULElement('toolbarbutton') : + document.createElement('button'); + button.id = 'sortana-reason-button'; + button.setAttribute('label', 'Show Reasoning'); + button.className = 'toolbarbutton-1'; + const icon = browser.runtime.getURL('resources/img/brain.png'); + if (button.setAttribute) { + button.setAttribute('image', icon); + } else { + button.style.backgroundImage = `url(${icon})`; + button.style.backgroundSize = 'contain'; + } + button.addEventListener('command', async () => { + const tabs = await browser.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs[0]?.id; + const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; + if (!msgs.length) return; + const url = browser.runtime.getURL(`reasoning.html?mid=${msgs[0].id}`); + browser.tabs.create({ url }); + }); + toolbar.appendChild(button); + } + if (document.readyState === 'complete' || document.readyState === 'interactive') { + addButton(); + } else { + document.addEventListener('DOMContentLoaded', addButton, { once: true }); + } +})(); From 004815e39a06e8160388d88d8b0586b23d6a92cb Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 01:39:36 -0500 Subject: [PATCH 003/124] Delete resources/img/brain.png --- resources/img/brain.png | 44 ----------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 resources/img/brain.png diff --git a/resources/img/brain.png b/resources/img/brain.png deleted file mode 100644 index 7d9e7d6..0000000 --- a/resources/img/brain.png +++ /dev/null @@ -1,44 +0,0 @@ - - - - -Object not found! - - - - - -

Object not found!

-

- - - The requested URL was not found on this server. - - - - If you entered the URL manually please check your - spelling and try again. - - - -

-

-If you think this is a server error, please contact -the webmaster. - -

- -

Error 404

-
- openmoji.org
- Apache -
- - - From 03844a5ab371feec4f74587b734ef12d55e7cfa4 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 02:02:55 -0500 Subject: [PATCH 004/124] Update ai-filter.sln --- ai-filter.sln | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ai-filter.sln b/ai-filter.sln index 6aeca89..77a10cb 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -13,6 +13,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution logger.js = logger.js manifest.json = manifest.json README.md = README.md + reasoning.html = reasoning.html + reasoning.js = reasoning.js EndProjectSection ProjectSection(FolderGlobals) = preProject Q_5_4Users_4Jordan_4Documents_4Gitea_4thunderbird-ai-filter_4src_4manifest_1json__JsonSchema = @@ -51,9 +53,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_ EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{68A87938-5C2B-49F5-8AAA-8A34FBBFD854}" + ProjectSection(SolutionItems) = preProject + resources\clearCacheButton.js = resources\clearCacheButton.js + resources\reasonButton.js = resources\reasonButton.js + EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-1755-4A95-A11B-6C90C701C5BF}" ProjectSection(SolutionItems) = preProject + resources\img\brain.png = resources\img\brain.png + resources\img\busy.png = resources\img\busy.png + resources\img\done.png = resources\img\done.png + resources\img\error.png = resources\img\error.png resources\img\full-logo.png = resources\img\full-logo.png resources\img\logo.png = resources\img\logo.png resources\img\logo128.png = resources\img\logo128.png From 22ab445dea92ca507956cfd62851474bb3a43b74 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 02:12:34 -0500 Subject: [PATCH 005/124] Add View Reasoning menu and fix button --- background.js | 29 ++++++++++++++++++++++++++--- resources/reasonButton.js | 2 +- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/background.js b/background.js index 17154b0..5b536a5 100644 --- a/background.js +++ b/background.js @@ -228,12 +228,17 @@ async function clearCacheForMessages(idsInput) { browser.messageDisplayAction.setLabel({ label: "Classify" }); } } - if (browser.messageDisplayScripts?.registerScripts) { + if (browser.messageDisplayScripts) { try { - await browser.messageDisplayScripts.registerScripts([ + const scripts = [ { js: [browser.runtime.getURL("resources/clearCacheButton.js")] }, { js: [browser.runtime.getURL("resources/reasonButton.js")] } - ]); + ]; + if (browser.messageDisplayScripts.registerScripts) { + await browser.messageDisplayScripts.registerScripts(scripts); + } else if (browser.messageDisplayScripts.register) { + await browser.messageDisplayScripts.register(scripts); + } } catch (e) { logger.aiLog("failed to register message display script", { level: 'warn' }, e); } @@ -259,6 +264,18 @@ async function clearCacheForMessages(idsInput) { title: "Clear AI Cache", contexts: ["message_display_action"], }); + browser.menus.create({ + id: "view-ai-reason-list", + title: "View Reasoning", + contexts: ["message_list"], + icons: { "16": "resources/img/brain.png" } + }); + browser.menus.create({ + id: "view-ai-reason-display", + title: "View Reasoning", + contexts: ["message_display_action"], + icons: { "16": "resources/img/brain.png" } + }); if (browser.messageDisplayAction) { browser.messageDisplayAction.onClicked.addListener(async (tab) => { @@ -281,6 +298,12 @@ async function clearCacheForMessages(idsInput) { const ids = info.selectedMessages?.messages?.map(m => m.id) || (info.messageId ? [info.messageId] : []); await clearCacheForMessages(ids); + } else if (info.menuItemId === "view-ai-reason-list" || info.menuItemId === "view-ai-reason-display") { + const id = info.messageId || info.selectedMessages?.messages?.[0]?.id; + if (id) { + const url = browser.runtime.getURL(`reasoning.html?mid=${id}`); + browser.tabs.create({ url }); + } } }); diff --git a/resources/reasonButton.js b/resources/reasonButton.js index 0206ab9..d0b38e7 100644 --- a/resources/reasonButton.js +++ b/resources/reasonButton.js @@ -7,7 +7,7 @@ document.createXULElement('toolbarbutton') : document.createElement('button'); button.id = 'sortana-reason-button'; - button.setAttribute('label', 'Show Reasoning'); + button.setAttribute('label', 'View Reasoning'); button.className = 'toolbarbutton-1'; const icon = browser.runtime.getURL('resources/img/brain.png'); if (button.setAttribute) { From a29a6299efc03b0ea049a34f72dd1db16022bfef Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 02:52:49 -0500 Subject: [PATCH 006/124] Use new scripting API for message display scripts --- background.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/background.js b/background.js index 5b536a5..ca235c8 100644 --- a/background.js +++ b/background.js @@ -228,11 +228,27 @@ async function clearCacheForMessages(idsInput) { browser.messageDisplayAction.setLabel({ label: "Classify" }); } } - if (browser.messageDisplayScripts) { + if (browser.scripting && browser.scripting.messageDisplay) { + try { + const scripts = [ + { + id: "clear-cache-button", + js: [browser.runtime.getURL("resources/clearCacheButton.js")], + }, + { + id: "reason-button", + js: [browser.runtime.getURL("resources/reasonButton.js")], + }, + ]; + await browser.scripting.messageDisplay.registerScripts(scripts); + } catch (e) { + logger.aiLog("failed to register message display script", { level: 'warn' }, e); + } + } else if (browser.messageDisplayScripts) { try { const scripts = [ { js: [browser.runtime.getURL("resources/clearCacheButton.js")] }, - { js: [browser.runtime.getURL("resources/reasonButton.js")] } + { js: [browser.runtime.getURL("resources/reasonButton.js")] }, ]; if (browser.messageDisplayScripts.registerScripts) { await browser.messageDisplayScripts.registerScripts(scripts); From 9ae22ad5ed209fc0ab1946e008dd78646e830895 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 02:52:49 -0500 Subject: [PATCH 007/124] Update README.md --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index e31a766..299ecdc 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,15 @@ Sortana requests the following Thunderbird permissions: - `accountsRead` – list accounts and folders for move actions. - `menus` – add context menu commands. +## Thunderbird Add-on Store Disclosures + +The [Third Party Library Usage](https://extensionworkshop.com/documentation/publish/third-party-library-usage/) policy +requires disclosure of third party libraries that are included in the add-on. Even though +the disclosure is only required for add-on review, they'll be listed here as well. Sortana +uses the following third party libraries: + +- [Bulma.css v1.0.4](https://github.com/jgthms/bulma/blob/1.0.4/css/bulma.css) + ## License This project is licensed under the terms of the GNU General Public License From a7009d644c8bd2502a400e6f0b01a95f592b6090 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 03:12:32 -0500 Subject: [PATCH 008/124] Fix script registration with relative URLs --- background.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/background.js b/background.js index ca235c8..1b25492 100644 --- a/background.js +++ b/background.js @@ -233,11 +233,11 @@ async function clearCacheForMessages(idsInput) { const scripts = [ { id: "clear-cache-button", - js: [browser.runtime.getURL("resources/clearCacheButton.js")], + js: ["resources/clearCacheButton.js"], }, { id: "reason-button", - js: [browser.runtime.getURL("resources/reasonButton.js")], + js: ["resources/reasonButton.js"], }, ]; await browser.scripting.messageDisplay.registerScripts(scripts); @@ -247,8 +247,8 @@ async function clearCacheForMessages(idsInput) { } else if (browser.messageDisplayScripts) { try { const scripts = [ - { js: [browser.runtime.getURL("resources/clearCacheButton.js")] }, - { js: [browser.runtime.getURL("resources/reasonButton.js")] }, + { js: ["resources/clearCacheButton.js"] }, + { js: ["resources/reasonButton.js"] }, ]; if (browser.messageDisplayScripts.registerScripts) { await browser.messageDisplayScripts.registerScripts(scripts); From 7be3c46e627ae0ebc680dd1d480a2742081743da Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 03:14:57 -0500 Subject: [PATCH 009/124] Update manifest.json --- manifest.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 0e9b20a..3c0b02b 100644 --- a/manifest.json +++ b/manifest.json @@ -38,6 +38,7 @@ "messagesUpdate", "messagesTagsList", "accountsRead", - "menus" + "menus", + "scripting" ] } From 437616900192faed1c738bfb14980046de73c777 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 03:55:47 -0500 Subject: [PATCH 010/124] Update AGENTS.md --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index e4c3c8b..87cfd6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ Additional documentation exists outside this repository. - Development guide: [Webextention-API for Thunderbird](https://webextension-api.thunderbird.net/en/stable/) - [Messages API](https://webextension-api.thunderbird.net/en/stable/messages.html) - [Message Tags API](https://webextension-api.thunderbird.net/en/stable/messages.tags.html) + - [messageDisplayAction API](https://webextension-api.thunderbird.net/en/stable/messageDisplayAction.html) - [Storage API](https://webextension-api.thunderbird.net/en/stable/storage.html) - Thunderbird Add-on Store Policies - [Third Party Library Usage](https://extensionworkshop.com/documentation/publish/third-party-library-usage/) From 1d7f0d534473c683b95151239be05827aec148df Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 04:08:04 -0500 Subject: [PATCH 011/124] Add message details popup --- ai-filter.sln | 4 -- background.js | 90 +++++++++++++++++----------------- reasoning.html => details.html | 7 ++- reasoning.js => details.js | 18 +++++-- manifest.json | 7 +-- resources/clearCacheButton.js | 22 --------- resources/reasonButton.js | 34 ------------- 7 files changed, 66 insertions(+), 116 deletions(-) rename reasoning.html => details.html (61%) rename reasoning.js => details.js (54%) delete mode 100644 resources/clearCacheButton.js delete mode 100644 resources/reasonButton.js diff --git a/ai-filter.sln b/ai-filter.sln index 77a10cb..960bee0 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -13,8 +13,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution logger.js = logger.js manifest.json = manifest.json README.md = README.md - reasoning.html = reasoning.html - reasoning.js = reasoning.js EndProjectSection ProjectSection(FolderGlobals) = preProject Q_5_4Users_4Jordan_4Documents_4Gitea_4thunderbird-ai-filter_4src_4manifest_1json__JsonSchema = @@ -54,8 +52,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{68A87938-5C2B-49F5-8AAA-8A34FBBFD854}" ProjectSection(SolutionItems) = preProject - resources\clearCacheButton.js = resources\clearCacheButton.js - resources\reasonButton.js = resources\reasonButton.js EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-1755-4A95-A11B-6C90C701C5BF}" diff --git a/background.js b/background.js index 1b25492..8e235d4 100644 --- a/background.js +++ b/background.js @@ -30,7 +30,7 @@ function setIcon(path) { } function updateActionIcon() { - let path = "resources/img/logo32.png"; + let path = "resources/img/brain.png"; if (processing || queuedCount > 0) { path = "resources/img/busy.png"; } @@ -223,40 +223,9 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("background.js loaded – ready to classify", {debug: true}); if (browser.messageDisplayAction) { - browser.messageDisplayAction.setTitle({ title: "Classify" }); + browser.messageDisplayAction.setTitle({ title: "Details" }); if (browser.messageDisplayAction.setLabel) { - browser.messageDisplayAction.setLabel({ label: "Classify" }); - } - } - if (browser.scripting && browser.scripting.messageDisplay) { - try { - const scripts = [ - { - id: "clear-cache-button", - js: ["resources/clearCacheButton.js"], - }, - { - id: "reason-button", - js: ["resources/reasonButton.js"], - }, - ]; - await browser.scripting.messageDisplay.registerScripts(scripts); - } catch (e) { - logger.aiLog("failed to register message display script", { level: 'warn' }, e); - } - } else if (browser.messageDisplayScripts) { - try { - const scripts = [ - { js: ["resources/clearCacheButton.js"] }, - { js: ["resources/reasonButton.js"] }, - ]; - if (browser.messageDisplayScripts.registerScripts) { - await browser.messageDisplayScripts.registerScripts(scripts); - } else if (browser.messageDisplayScripts.register) { - await browser.messageDisplayScripts.register(scripts); - } - } catch (e) { - logger.aiLog("failed to register message display script", { level: 'warn' }, e); + browser.messageDisplayAction.setLabel({ label: "Details" }); } } @@ -293,17 +262,7 @@ async function clearCacheForMessages(idsInput) { icons: { "16": "resources/img/brain.png" } }); - if (browser.messageDisplayAction) { - browser.messageDisplayAction.onClicked.addListener(async (tab) => { - try { - const msgs = await browser.messageDisplay.getDisplayedMessages(tab.id); - const ids = msgs.map(m => m.id); - await applyAiRules(ids); - } catch (e) { - logger.aiLog("failed to apply AI rules from action", { level: 'error' }, e); - } - }); - } + browser.menus.onClicked.addListener(async info => { if (info.menuItemId === "apply-ai-rules-list" || info.menuItemId === "apply-ai-rules-display") { @@ -317,7 +276,7 @@ async function clearCacheForMessages(idsInput) { } else if (info.menuItemId === "view-ai-reason-list" || info.menuItemId === "view-ai-reason-display") { const id = info.messageId || info.selectedMessages?.messages?.[0]?.id; if (id) { - const url = browser.runtime.getURL(`reasoning.html?mid=${id}`); + const url = browser.runtime.getURL(`details.html?mid=${id}`); browser.tabs.create({ url }); } } @@ -383,6 +342,45 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("failed to collect reasons", { level: 'error' }, e); return { subject: '', reasons: [] }; } + } else if (msg?.type === "sortana:getDetails") { + try { + const id = msg.id; + const hdr = await messenger.messages.get(id); + const subject = hdr?.subject || ""; + if (!aiRules.length) { + const { aiRules: stored } = await storage.local.get("aiRules"); + aiRules = Array.isArray(stored) ? stored.map(r => { + if (r.actions) return r; + const actions = []; + if (r.tag) actions.push({ type: 'tag', tagKey: r.tag }); + if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); + const rule = { criterion: r.criterion, actions }; + if (r.stopProcessing) rule.stopProcessing = true; + return rule; + }) : []; + } + const results = []; + for (const rule of aiRules) { + const key = await sha256Hex(`${id}|${rule.criterion}`); + const matched = AiClassifier.getCachedResult(key); + const reason = AiClassifier.getReason(key); + if (matched !== null || reason) { + results.push({ criterion: rule.criterion, matched, reason }); + } + } + return { subject, results }; + } catch (e) { + logger.aiLog("failed to collect details", { level: 'error' }, e); + return { subject: '', results: [] }; + } + } else if (msg?.type === "sortana:clearCacheForMessage") { + try { + await clearCacheForMessages([msg.id]); + return { ok: true }; + } catch (e) { + logger.aiLog("failed to clear cache for message", { level: 'error' }, e); + return { ok: false }; + } } else { logger.aiLog("Unknown message type, ignoring", {level: 'warn'}, msg?.type); } diff --git a/reasoning.html b/details.html similarity index 61% rename from reasoning.html rename to details.html index bb1577d..1502471 100644 --- a/reasoning.html +++ b/details.html @@ -2,7 +2,7 @@ - AI Reasoning + AI Details @@ -10,8 +10,11 @@

+
+ +
- + diff --git a/reasoning.js b/details.js similarity index 54% rename from reasoning.js rename to details.js index eb0b9da..2b5b9e7 100644 --- a/reasoning.js +++ b/details.js @@ -3,25 +3,33 @@ document.addEventListener('DOMContentLoaded', async () => { const id = parseInt(params.get('mid'), 10); if (!id) return; try { - const { subject, reasons } = await browser.runtime.sendMessage({ type: 'sortana:getReasons', id }); + const { subject, results } = await browser.runtime.sendMessage({ type: 'sortana:getDetails', id }); document.getElementById('subject').textContent = subject; const container = document.getElementById('rules'); - for (const r of reasons) { + for (const r of results) { const article = document.createElement('article'); - article.className = 'message mb-4'; + const color = r.matched === true ? 'is-success' : 'is-danger'; + article.className = `message ${color} mb-4`; const header = document.createElement('div'); header.className = 'message-header'; header.innerHTML = `

${r.criterion}

`; const body = document.createElement('div'); body.className = 'message-body'; + const status = document.createElement('p'); + status.textContent = r.matched ? 'Matched' : 'Did not match'; const pre = document.createElement('pre'); - pre.textContent = r.reason; + pre.textContent = r.reason || ''; + body.appendChild(status); body.appendChild(pre); article.appendChild(header); article.appendChild(body); container.appendChild(article); } + document.getElementById('clear').addEventListener('click', async () => { + await browser.runtime.sendMessage({ type: 'sortana:clearCacheForMessage', id }); + window.close(); + }); } catch (e) { - console.error('failed to load reasons', e); + console.error('failed to load details', e); } }); diff --git a/manifest.json b/manifest.json index 3c0b02b..3f17845 100644 --- a/manifest.json +++ b/manifest.json @@ -22,9 +22,10 @@ "default_icon": "resources/img/logo32.png" }, "message_display_action": { - "default_icon": "resources/img/logo32.png", - "default_title": "Classify", - "default_label": "Classify" + "default_icon": "resources/img/brain.png", + "default_title": "Details", + "default_label": "Details", + "default_popup": "details.html" }, "background": { "scripts": [ "background.js" ] }, "options_ui": { diff --git a/resources/clearCacheButton.js b/resources/clearCacheButton.js deleted file mode 100644 index a4d2adb..0000000 --- a/resources/clearCacheButton.js +++ /dev/null @@ -1,22 +0,0 @@ -(function() { - function addButton() { - const toolbar = document.querySelector("#header-view-toolbar") || - document.querySelector("#mail-toolbox toolbar"); - if (!toolbar || document.getElementById('sortana-clear-cache-button')) return; - const button = document.createXULElement ? - document.createXULElement('toolbarbutton') : - document.createElement('button'); - button.id = 'sortana-clear-cache-button'; - button.setAttribute('label', 'Clear Cache'); - button.className = 'toolbarbutton-1'; - button.addEventListener('command', () => { - browser.runtime.sendMessage({ type: 'sortana:clearCacheForDisplayed' }); - }); - toolbar.appendChild(button); - } - if (document.readyState === 'complete' || document.readyState === 'interactive') { - addButton(); - } else { - document.addEventListener('DOMContentLoaded', addButton, { once: true }); - } -})(); diff --git a/resources/reasonButton.js b/resources/reasonButton.js deleted file mode 100644 index d0b38e7..0000000 --- a/resources/reasonButton.js +++ /dev/null @@ -1,34 +0,0 @@ -(function() { - function addButton() { - const toolbar = document.querySelector("#header-view-toolbar") || - document.querySelector("#mail-toolbox toolbar"); - if (!toolbar || document.getElementById('sortana-reason-button')) return; - const button = document.createXULElement ? - document.createXULElement('toolbarbutton') : - document.createElement('button'); - button.id = 'sortana-reason-button'; - button.setAttribute('label', 'View Reasoning'); - button.className = 'toolbarbutton-1'; - const icon = browser.runtime.getURL('resources/img/brain.png'); - if (button.setAttribute) { - button.setAttribute('image', icon); - } else { - button.style.backgroundImage = `url(${icon})`; - button.style.backgroundSize = 'contain'; - } - button.addEventListener('command', async () => { - const tabs = await browser.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs[0]?.id; - const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; - if (!msgs.length) return; - const url = browser.runtime.getURL(`reasoning.html?mid=${msgs[0].id}`); - browser.tabs.create({ url }); - }); - toolbar.appendChild(button); - } - if (document.readyState === 'complete' || document.readyState === 'interactive') { - addButton(); - } else { - document.addEventListener('DOMContentLoaded', addButton, { once: true }); - } -})(); From 9b91abb338d17147792c3ca89dc30102afaebccd Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 21:59:11 -0500 Subject: [PATCH 012/124] Fix details popup when no mid parameter --- details.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/details.js b/details.js index 2b5b9e7..0d190d5 100644 --- a/details.js +++ b/details.js @@ -1,6 +1,17 @@ document.addEventListener('DOMContentLoaded', async () => { const params = new URLSearchParams(location.search); - const id = parseInt(params.get('mid'), 10); + let id = parseInt(params.get('mid'), 10); + + if (!id) { + try { + const tabs = await browser.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs[0]?.id; + const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; + id = msgs[0]?.id; + } catch (e) { + console.error('failed to determine message id', e); + } + } if (!id) return; try { const { subject, results } = await browser.runtime.sendMessage({ type: 'sortana:getDetails', id }); From ef864651d4073e8b7f1fee2c8e5da83c32d32858 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 28 Jun 2025 02:56:29 -0500 Subject: [PATCH 013/124] Update ai-filter.sln --- ai-filter.sln | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ai-filter.sln b/ai-filter.sln index 960bee0..40cec4f 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -9,6 +9,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution AGENTS.md = AGENTS.md background.js = background.js build-xpi.ps1 = build-xpi.ps1 + details.html = details.html + details.js = details.js LICENSE = LICENSE logger.js = logger.js manifest.json = manifest.json @@ -51,8 +53,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_ EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{68A87938-5C2B-49F5-8AAA-8A34FBBFD854}" - ProjectSection(SolutionItems) = preProject - EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-1755-4A95-A11B-6C90C701C5BF}" ProjectSection(SolutionItems) = preProject From bced7447b254327c18dfd3514cb898b4a80bf757 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 27 Jun 2025 22:20:38 -0500 Subject: [PATCH 014/124] Export getCachedResult from classifier --- modules/AiClassifier.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 4180973..cef1842 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -387,4 +387,4 @@ async function classifyText(text, criterion, cacheKey = null) { } } -export { classifyText, classifyTextSync, setConfig, removeCacheEntries, getReason }; +export { classifyText, classifyTextSync, setConfig, removeCacheEntries, getReason, getCachedResult }; From d69d0cae669ca96bcb4e1b584b1cc77c268f78a2 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 28 Jun 2025 15:46:30 -0500 Subject: [PATCH 015/124] Merge AI caches and add cache key helper --- AGENTS.md | 7 ++ README.md | 9 +- background.js | 13 +-- modules/AiClassifier.js | 155 ++++++++++++++--------------- modules/ExpressionSearchFilter.jsm | 11 +- 5 files changed, 95 insertions(+), 100 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 87cfd6f..ad83d34 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,3 +59,10 @@ base64 data should be replaced with placeholders showing the byte size. The final string should have the headers, a brief attachment section, then the plain text extracted from all text parts. +### Cache Strategy + +`aiCache` persists classification results. Each key is the SHA‑256 hex of +`"|"` and maps to an object with `matched` and `reason` +properties. Any legacy `aiReasonCache` data is merged into `aiCache` the first +time the add-on loads after an update. + diff --git a/README.md b/README.md index 299ecdc..ed362eb 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ message meets a specified criterion. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. - **Filter editor integration** – patches Thunderbird's filter editor to accept text criteria for AI classification. -- **Persistent result caching** – classification results are saved to disk so messages aren't re-evaluated across restarts. +- **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. - **Automatic rules** – create rules that tag or move new messages based on AI classification. @@ -25,6 +25,13 @@ message meets a specified criterion. - **Status icons** – toolbar icons show when classification is in progress and briefly display success or error states. - **Packaging script** – `build-xpi.ps1` builds an XPI ready for installation. +### Cache Storage + +Classification results are stored under the `aiCache` key in extension storage. +Each entry maps a SHA‑256 hash of `"|"` to an object +containing `matched` and `reason` fields. Older installations with a separate +`aiReasonCache` will be migrated automatically on startup. + ## Architecture Overview Sortana is implemented entirely with standard WebExtension scripts—no custom experiment code is required: diff --git a/background.js b/background.js index 8e235d4..ab37701 100644 --- a/background.js +++ b/background.js @@ -43,10 +43,6 @@ function showTransientIcon(path, delay = 1500) { iconTimer = setTimeout(updateActionIcon, delay); } -async function sha256Hex(str) { - const buf = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(str)); - return Array.from(new Uint8Array(buf), b => b.toString(16).padStart(2, '0')).join(''); -} function byteSize(str) { return new TextEncoder().encode(str || "").length; @@ -117,7 +113,7 @@ async function applyAiRules(idsInput) { const text = buildEmailText(full); for (const rule of aiRules) { - const cacheKey = await sha256Hex(`${id}|${rule.criterion}`); + const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion); const matched = await AiClassifier.classifyText(text, rule.criterion, cacheKey); if (matched) { for (const act of (rule.actions || [])) { @@ -168,7 +164,7 @@ async function clearCacheForMessages(idsInput) { for (const msg of ids) { const id = msg?.id ?? msg; for (const rule of aiRules) { - const key = await sha256Hex(`${id}|${rule.criterion}`); + const key = await AiClassifier.buildCacheKey(id, rule.criterion); keys.push(key); } } @@ -192,6 +188,7 @@ async function clearCacheForMessages(idsInput) { const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "aiRules"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); + await AiClassifier.init(); aiRules = Array.isArray(store.aiRules) ? store.aiRules.map(r => { if (r.actions) return r; const actions = []; @@ -331,7 +328,7 @@ async function clearCacheForMessages(idsInput) { } const reasons = []; for (const rule of aiRules) { - const key = await sha256Hex(`${id}|${rule.criterion}`); + const key = await AiClassifier.buildCacheKey(id, rule.criterion); const reason = AiClassifier.getReason(key); if (reason) { reasons.push({ criterion: rule.criterion, reason }); @@ -361,7 +358,7 @@ async function clearCacheForMessages(idsInput) { } const results = []; for (const rule of aiRules) { - const key = await sha256Hex(`${id}|${rule.criterion}`); + const key = await AiClassifier.buildCacheKey(id, rule.criterion); const matched = AiClassifier.getCachedResult(key); const reason = AiClassifier.getReason(key); if (matched !== null || reason) { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index cef1842..e123288 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -49,8 +49,39 @@ let gAiParams = { let gCache = new Map(); let gCacheLoaded = false; -let gReasonCache = new Map(); -let gReasonCacheLoaded = false; + +function sha256HexSync(str) { + try { + const hasher = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); + hasher.init(Ci.nsICryptoHash.SHA256); + const data = new TextEncoder().encode(str); + hasher.update(data, data.length); + const binary = hasher.finish(false); + return Array.from(binary, c => ("0" + c.charCodeAt(0).toString(16)).slice(-2)).join(""); + } catch (e) { + aiLog(`sha256HexSync failed`, { level: 'error' }, e); + return ""; + } +} + +async function sha256Hex(str) { + if (typeof crypto?.subtle?.digest === "function") { + const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(str)); + return Array.from(new Uint8Array(buf), b => b.toString(16).padStart(2, "0")).join(""); + } + return sha256HexSync(str); +} + +function buildCacheKeySync(id, criterion) { + return sha256HexSync(`${id}|${criterion}`); +} + +async function buildCacheKey(id, criterion) { + if (Services) { + return buildCacheKeySync(id, criterion); + } + return sha256Hex(`${id}|${criterion}`); +} async function loadCache() { if (gCacheLoaded) { @@ -58,16 +89,29 @@ async function loadCache() { } aiLog(`[AiClassifier] Loading cache`, {debug: true}); try { - const { aiCache } = await storage.local.get("aiCache"); + const { aiCache, aiReasonCache } = await storage.local.get(["aiCache", "aiReasonCache"]); if (aiCache) { for (let [k, v] of Object.entries(aiCache)) { - aiLog(`[AiClassifier] ⮡ Loaded entry '${k}' → ${v}`, {debug: true}); - gCache.set(k, v); + if (v && typeof v === "object") { + gCache.set(k, { matched: v.matched ?? null, reason: v.reason || "" }); + } else { + gCache.set(k, { matched: v, reason: "" }); + } } aiLog(`[AiClassifier] Loaded ${gCache.size} cache entries`, {debug: true}); } else { aiLog(`[AiClassifier] Cache is empty`, {debug: true}); } + if (aiReasonCache) { + aiLog(`[AiClassifier] Migrating ${Object.keys(aiReasonCache).length} reason entries`, {debug: true}); + for (let [k, reason] of Object.entries(aiReasonCache)) { + let entry = gCache.get(k) || { matched: null, reason: "" }; + entry.reason = reason; + gCache.set(k, entry); + } + await storage.local.remove("aiReasonCache"); + await storage.local.set({ aiCache: Object.fromEntries(gCache) }); + } } catch (e) { aiLog(`Failed to load cache`, {level: 'error'}, e); } @@ -96,49 +140,6 @@ async function saveCache(updatedKey, updatedValue) { } } -async function loadReasonCache() { - if (gReasonCacheLoaded) { - return; - } - aiLog(`[AiClassifier] Loading reason cache`, {debug: true}); - try { - const { aiReasonCache } = await storage.local.get("aiReasonCache"); - if (aiReasonCache) { - for (let [k, v] of Object.entries(aiReasonCache)) { - aiLog(`[AiClassifier] ⮡ Loaded reason '${k}'`, {debug: true}); - gReasonCache.set(k, v); - } - aiLog(`[AiClassifier] Loaded ${gReasonCache.size} reason entries`, {debug: true}); - } else { - aiLog(`[AiClassifier] Reason cache is empty`, {debug: true}); - } - } catch (e) { - aiLog(`Failed to load reason cache`, {level: 'error'}, e); - } - gReasonCacheLoaded = true; -} - -function loadReasonCacheSync() { - if (!gReasonCacheLoaded) { - if (!Services?.tm?.spinEventLoopUntil) { - throw new Error("loadReasonCacheSync requires Services"); - } - let done = false; - loadReasonCache().finally(() => { done = true; }); - Services.tm.spinEventLoopUntil(() => done); - } -} - -async function saveReasonCache(updatedKey, updatedValue) { - if (typeof updatedKey !== "undefined") { - aiLog(`[AiClassifier] ⮡ Persisting reason '${updatedKey}'`, {debug: true}); - } - try { - await storage.local.set({ aiReasonCache: Object.fromEntries(gReasonCache) }); - } catch (e) { - aiLog(`Failed to save reason cache`, {level: 'error'}, e); - } -} async function loadTemplate(name) { try { @@ -220,26 +221,27 @@ function getCachedResult(cacheKey) { if (Services?.tm?.spinEventLoopUntil) { loadCacheSync(); } else { - // In non-privileged contexts we can't block, so bail out early. return null; } } if (cacheKey && gCache.has(cacheKey)) { aiLog(`[AiClassifier] Cache hit for key: ${cacheKey}`, {debug: true}); - return gCache.get(cacheKey); + const entry = gCache.get(cacheKey); + return entry?.matched ?? null; } return null; } function getReason(cacheKey) { - if (!gReasonCacheLoaded) { + if (!gCacheLoaded) { if (Services?.tm?.spinEventLoopUntil) { - loadReasonCacheSync(); + loadCacheSync(); } else { return null; } } - return cacheKey ? gReasonCache.get(cacheKey) || null : null; + const entry = gCache.get(cacheKey); + return cacheKey && entry ? entry.reason || null : null; } function buildPayload(text, criterion) { @@ -260,20 +262,20 @@ function parseMatch(result) { return { matched, reason: thinkText }; } -function cacheResult(cacheKey, matched) { - if (cacheKey) { - aiLog(`[AiClassifier] Caching entry '${cacheKey}' → ${matched}`, {debug: true}); - gCache.set(cacheKey, matched); - saveCache(cacheKey, matched); +function cacheEntry(cacheKey, matched, reason) { + if (!cacheKey) { + return; } -} - -function cacheReason(cacheKey, reason) { - if (cacheKey) { - aiLog(`[AiClassifier] Caching reason '${cacheKey}'`, {debug: true}); - gReasonCache.set(cacheKey, reason); - saveReasonCache(cacheKey, reason); + aiLog(`[AiClassifier] Caching entry '${cacheKey}'`, {debug: true}); + const entry = gCache.get(cacheKey) || { matched: null, reason: "" }; + if (typeof matched === "boolean") { + entry.matched = matched; } + if (typeof reason === "string") { + entry.reason = reason; + } + gCache.set(cacheKey, entry); + saveCache(cacheKey, entry); } async function removeCacheEntries(keys = []) { @@ -289,14 +291,9 @@ async function removeCacheEntries(keys = []) { removed = true; aiLog(`[AiClassifier] Removed cache entry '${key}'`, {debug: true}); } - if (gReasonCache.delete(key)) { - removed = true; - aiLog(`[AiClassifier] Removed reason entry '${key}'`, {debug: true}); - } } if (removed) { await saveCache(); - await saveReasonCache(); } } @@ -304,9 +301,6 @@ function classifyTextSync(text, criterion, cacheKey = null) { if (!Services?.tm?.spinEventLoopUntil) { throw new Error("classifyTextSync requires Services"); } - if (!gReasonCacheLoaded) { - loadReasonCacheSync(); - } const cached = getCachedResult(cacheKey); if (cached !== null) { return cached; @@ -329,8 +323,7 @@ function classifyTextSync(text, criterion, cacheKey = null) { const json = await response.json(); aiLog(`[AiClassifier] Received response:`, {debug: true}, json); result = parseMatch(json); - cacheResult(cacheKey, result.matched); - cacheReason(cacheKey, result.reason); + cacheEntry(cacheKey, result.matched, result.reason); result = result.matched; } else { aiLog(`HTTP status ${response.status}`, {level: 'warn'}); @@ -351,9 +344,6 @@ async function classifyText(text, criterion, cacheKey = null) { if (!gCacheLoaded) { await loadCache(); } - if (!gReasonCacheLoaded) { - await loadReasonCache(); - } const cached = getCachedResult(cacheKey); if (cached !== null) { return cached; @@ -378,8 +368,7 @@ async function classifyText(text, criterion, cacheKey = null) { const result = await response.json(); aiLog(`[AiClassifier] Received response:`, {debug: true}, result); const parsed = parseMatch(result); - cacheResult(cacheKey, parsed.matched); - cacheReason(cacheKey, parsed.reason); + cacheEntry(cacheKey, parsed.matched, parsed.reason); return parsed.matched; } catch (e) { aiLog(`HTTP request failed`, {level: 'error'}, e); @@ -387,4 +376,8 @@ async function classifyText(text, criterion, cacheKey = null) { } } -export { classifyText, classifyTextSync, setConfig, removeCacheEntries, getReason, getCachedResult }; +async function init() { + await loadCache(); +} + +export { classifyText, classifyTextSync, setConfig, removeCacheEntries, getReason, getCachedResult, buildCacheKey, buildCacheKeySync, init }; diff --git a/modules/ExpressionSearchFilter.jsm b/modules/ExpressionSearchFilter.jsm index 791c181..b9998a9 100644 --- a/modules/ExpressionSearchFilter.jsm +++ b/modules/ExpressionSearchFilter.jsm @@ -5,15 +5,6 @@ var { aiLog } = ChromeUtils.import("resource://aifilter/modules/logger.jsm"); var AiClassifier = ChromeUtils.importESModule("resource://aifilter/modules/AiClassifier.js"); var { getPlainText } = ChromeUtils.import("resource://aifilter/modules/messageUtils.jsm"); -function sha256Hex(str) { - const hasher = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); - hasher.init(Ci.nsICryptoHash.SHA256); - const data = new TextEncoder().encode(str); - hasher.update(data, data.length); - const binary = hasher.finish(false); - return Array.from(binary, c => ("0" + c.charCodeAt(0).toString(16)).slice(-2)).join(""); -} - var EXPORTED_SYMBOLS = ["AIFilter", "ClassificationTerm"]; class CustomerTermBase { @@ -70,7 +61,7 @@ class ClassificationTerm extends CustomerTermBase { op === Ci.nsMsgSearchOp.DoesntMatch ? "doesn't match" : `unknown (${op})`; aiLog(`[ExpressionSearchFilter] Matching message ${msgHdr.messageId} using op "${opName}" and value "${value}"`, {debug: true}); - let key = [msgHdr.messageId, op, value].map(sha256Hex).join("|"); + let key = AiClassifier.buildCacheKeySync(msgHdr.messageId, value); let body = getPlainText(msgHdr); let matched = AiClassifier.classifyTextSync(body, value, key); From 3e1df7be3f4abd5f0cf4deb77115b4d305070831 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 00:07:37 -0500 Subject: [PATCH 016/124] Preserve existing tags when applying multiple rules --- background.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/background.js b/background.js index ab37701..b49e691 100644 --- a/background.js +++ b/background.js @@ -111,6 +111,13 @@ async function applyAiRules(idsInput) { try { const full = await messenger.messages.getFull(id); const text = buildEmailText(full); + let currentTags = []; + try { + const hdr = await messenger.messages.get(id); + currentTags = Array.isArray(hdr.tags) ? [...hdr.tags] : []; + } catch (e) { + currentTags = []; + } for (const rule of aiRules) { const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion); @@ -118,7 +125,10 @@ async function applyAiRules(idsInput) { if (matched) { for (const act of (rule.actions || [])) { if (act.type === 'tag' && act.tagKey) { - await messenger.messages.update(id, { tags: [act.tagKey] }); + if (!currentTags.includes(act.tagKey)) { + currentTags.push(act.tagKey); + await messenger.messages.update(id, { tags: currentTags }); + } } else if (act.type === 'move' && act.folder) { await messenger.messages.move([id], act.folder); } else if (act.type === 'junk') { From 41a0f4f8f252e98c959aabefad7e738dc37a0ee1 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 00:26:26 -0500 Subject: [PATCH 017/124] Added additional logging --- modules/AiClassifier.js | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index e123288..ff80aac 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -352,6 +352,7 @@ async function classifyText(text, criterion, cacheKey = null) { const payload = buildPayload(text, criterion); aiLog(`[AiClassifier] Sending classification request to ${gEndpoint}`, {debug: true}); + aiLog(`[AiClassifier] Classification request payload:`, { debug: true }, payload); try { const response = await fetch(gEndpoint, { From 8ba2a931b9d72b66b0dd483332fccf21f28a7982 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 02:37:05 -0500 Subject: [PATCH 018/124] Use Message-ID for cache keys --- AGENTS.md | 2 +- README.md | 2 +- modules/AiClassifier.js | 23 +++++++++++++++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ad83d34..9ae810b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ text extracted from all text parts. ### Cache Strategy `aiCache` persists classification results. Each key is the SHA‑256 hex of -`"|"` and maps to an object with `matched` and `reason` +`"|"` and maps to an object with `matched` and `reason` properties. Any legacy `aiReasonCache` data is merged into `aiCache` the first time the add-on loads after an update. diff --git a/README.md b/README.md index ed362eb..e6aecbe 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ message meets a specified criterion. ### Cache Storage Classification results are stored under the `aiCache` key in extension storage. -Each entry maps a SHA‑256 hash of `"|"` to an object +Each entry maps a SHA‑256 hash of `"|"` to an object containing `matched` and `reason` fields. Older installations with a separate `aiReasonCache` will be migrated automatically on startup. diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index ff80aac..a546fcd 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -76,11 +76,26 @@ function buildCacheKeySync(id, criterion) { return sha256HexSync(`${id}|${criterion}`); } -async function buildCacheKey(id, criterion) { - if (Services) { - return buildCacheKeySync(id, criterion); +async function resolveHeaderId(id) { + if (typeof id === "number" && typeof messenger?.messages?.get === "function") { + try { + const hdr = await messenger.messages.get(id); + if (hdr?.headerMessageId) { + return hdr.headerMessageId; + } + } catch (e) { + aiLog(`Failed to resolve headerMessageId for ${id}`, { level: 'warn' }, e); + } } - return sha256Hex(`${id}|${criterion}`); + return String(id); +} + +async function buildCacheKey(id, criterion) { + const resolvedId = await resolveHeaderId(id); + if (Services) { + return buildCacheKeySync(resolvedId, criterion); + } + return sha256Hex(`${resolvedId}|${criterion}`); } async function loadCache() { From a77e5e68fbf128312dbca2dde80c9af55757440c Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 02:42:58 -0500 Subject: [PATCH 019/124] clean up --- ai-filter.sln | 8 +-- modules/ExpressionSearchFilter.jsm | 90 ------------------------------ modules/messageUtils.jsm | 89 ----------------------------- 3 files changed, 2 insertions(+), 185 deletions(-) delete mode 100644 modules/ExpressionSearchFilter.jsm delete mode 100644 modules/messageUtils.jsm diff --git a/ai-filter.sln b/ai-filter.sln index 40cec4f..2d171dd 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -29,8 +29,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "options", "options", "{7372 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "modules", "modules", "{75ED3C1E-D3C7-4546-9F2E-AC85859DDF4B}" ProjectSection(SolutionItems) = preProject + modules\AiClassifier.js = modules\AiClassifier.js modules\ExpressionSearchFilter.jsm = modules\ExpressionSearchFilter.jsm modules\logger.jsm = modules\logger.jsm + modules\messageUtils.jsm = modules\messageUtils.jsm EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_locales", "_locales", "{D446E5C6-BDDE-4091-BD1A-EC57170003CF}" @@ -40,11 +42,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "en-US", "en-US", "{8BEA7793 _locales\en-US\messages.json = _locales\en-US\messages.json EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "content", "content", "{028FDA4B-AC3E-4A0E-9291-978E213F9B78}" - ProjectSection(SolutionItems) = preProject - content\filterEditor.js = content\filterEditor.js - EndProjectSection -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_templates", "{86516D53-50D4-4FE2-9D8A-977A8F5EBDBD}" ProjectSection(SolutionItems) = preProject prompt_templates\mistral.txt = prompt_templates\mistral.txt @@ -79,7 +76,6 @@ Global {75ED3C1E-D3C7-4546-9F2E-AC85859DDF4B} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {D446E5C6-BDDE-4091-BD1A-EC57170003CF} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {8BEA7793-3336-40ED-AB96-7FFB09FEB0F6} = {D446E5C6-BDDE-4091-BD1A-EC57170003CF} - {028FDA4B-AC3E-4A0E-9291-978E213F9B78} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {86516D53-50D4-4FE2-9D8A-977A8F5EBDBD} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {68A87938-5C2B-49F5-8AAA-8A34FBBFD854} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {F266602F-1755-4A95-A11B-6C90C701C5BF} = {68A87938-5C2B-49F5-8AAA-8A34FBBFD854} diff --git a/modules/ExpressionSearchFilter.jsm b/modules/ExpressionSearchFilter.jsm deleted file mode 100644 index b9998a9..0000000 --- a/modules/ExpressionSearchFilter.jsm +++ /dev/null @@ -1,90 +0,0 @@ -"use strict"; -var { ExtensionParent } = ChromeUtils.importESModule("resource://gre/modules/ExtensionParent.sys.mjs"); -var { MailServices } = ChromeUtils.importESModule("resource:///modules/MailServices.sys.mjs"); -var { aiLog } = ChromeUtils.import("resource://aifilter/modules/logger.jsm"); -var AiClassifier = ChromeUtils.importESModule("resource://aifilter/modules/AiClassifier.js"); -var { getPlainText } = ChromeUtils.import("resource://aifilter/modules/messageUtils.jsm"); - -var EXPORTED_SYMBOLS = ["AIFilter", "ClassificationTerm"]; - -class CustomerTermBase { - constructor(nameId, operators) { - // Lookup our extension instance using the ID from manifest.json - // so locale strings are resolved correctly. - this.extension = ExtensionParent.GlobalManager.getExtension("ai-filter@jordanwages"); - this.id = "aifilter#" + nameId; - this.name = this.extension.localeData.localizeMessage(nameId); - this.operators = operators; - - aiLog(`[ExpressionSearchFilter] Initialized term base "${this.id}"`, {debug: true}); - } - - - getEnabled() { - aiLog(`[ExpressionSearchFilter] getEnabled() called on "${this.id}"`, {debug: true}); - return true; - } - - getAvailable() { - aiLog(`[ExpressionSearchFilter] getAvailable() called on "${this.id}"`, {debug: true}); - return true; - } - - getAvailableOperators() { - aiLog(`[ExpressionSearchFilter] getAvailableOperators() called on "${this.id}"`, {debug: true}); - return this.operators; - } - - getAvailableValues() { - aiLog(`[ExpressionSearchFilter] getAvailableValues() called on "${this.id}"`, {debug: true}); - return null; - } - - get attrib() { - aiLog(`[ExpressionSearchFilter] attrib getter called for "${this.id}"`, {debug: true}); - - //return Ci.nsMsgSearchAttrib.Custom; - } -} - - -class ClassificationTerm extends CustomerTermBase { - constructor() { - super("classification", [Ci.nsMsgSearchOp.Matches, Ci.nsMsgSearchOp.DoesntMatch]); - aiLog(`[ExpressionSearchFilter] ClassificationTerm constructed`, {debug: true}); - } - - needsBody() { return true; } - - match(msgHdr, value, op) { - const opName = op === Ci.nsMsgSearchOp.Matches ? "matches" : - op === Ci.nsMsgSearchOp.DoesntMatch ? "doesn't match" : `unknown (${op})`; - aiLog(`[ExpressionSearchFilter] Matching message ${msgHdr.messageId} using op "${opName}" and value "${value}"`, {debug: true}); - - let key = AiClassifier.buildCacheKeySync(msgHdr.messageId, value); - let body = getPlainText(msgHdr); - - let matched = AiClassifier.classifyTextSync(body, value, key); - - if (op === Ci.nsMsgSearchOp.DoesntMatch) { - matched = !matched; - aiLog(`[ExpressionSearchFilter] Operator is "doesn't match" → inverting to ${matched}`, {debug: true}); - } - - aiLog(`[ExpressionSearchFilter] Final match result: ${matched}`, {debug: true}); - return matched; - } -} - -(function register() { - aiLog(`[ExpressionSearchFilter] Registering custom filter term...`, {debug: true}); - let term = new ClassificationTerm(); - if (!MailServices.filters.getCustomTerm(term.id)) { - MailServices.filters.addCustomTerm(term); - aiLog(`[ExpressionSearchFilter] Registered term: ${term.id}`, {debug: true}); - } else { - aiLog(`[ExpressionSearchFilter] Term already registered: ${term.id}`, {debug: true}); - } -})(); - -var AIFilter = { setConfig: AiClassifier.setConfig }; diff --git a/modules/messageUtils.jsm b/modules/messageUtils.jsm deleted file mode 100644 index a4978a7..0000000 --- a/modules/messageUtils.jsm +++ /dev/null @@ -1,89 +0,0 @@ -"use strict"; -var { NetUtil } = ChromeUtils.importESModule("resource://gre/modules/NetUtil.sys.mjs"); -var { MimeParser } = ChromeUtils.importESModule("resource:///modules/mimeParser.sys.mjs"); -var { aiLog } = ChromeUtils.import("resource://aifilter/modules/logger.jsm"); - -var EXPORTED_SYMBOLS = ["getPlainText"]; - -function getPlainText(msgHdr) { - aiLog(`[ExpressionSearchFilter] Extracting plain text for message ID ${msgHdr.messageId}`, {debug: true}); - let folder = msgHdr.folder; - if (!folder.getMsgInputStream) return ""; - let reusable = {}; - let stream = folder.getMsgInputStream(msgHdr, reusable); - let data = NetUtil.readInputStreamToString(stream, msgHdr.messageSize); - if (!reusable.value) stream.close(); - - let parser = Cc["@mozilla.org/parserutils;1"].getService(Ci.nsIParserUtils); - - try { - let root = MimeParser.parseSync(data, {strformat: "unicode"}); - let parts = []; - - function pushPlaceholder(type, info, bytes) { - bytes = bytes || 0; - let prettyType = type.split("/")[1] || type; - parts.push(`[${info}: ${prettyType}, ${bytes} bytes]`); - } - - function byteSizeFromBase64(str) { - let clean = str.replace(/[^A-Za-z0-9+/=]/g, ""); - return Math.floor(clean.length * 3 / 4); - } - - function replaceInlineBase64(text) { - return text.replace(/[A-Za-z0-9+/]{100,}={0,2}/g, - m => `[base64: ${byteSizeFromBase64(m)} bytes]`); - } - - function walk(node) { - if (node.parts && node.parts.length) { - for (let child of node.parts) { - walk(child); - } - return; - } - - let ct = (node.contentType || "text/plain").toLowerCase(); - let cd = (node.headers?.["content-disposition"]?.[0] || "").toLowerCase(); - let enc = (node.headers?.["content-transfer-encoding"]?.[0] || "").toLowerCase(); - let bodyText = String(node.body || ""); - - if (cd.includes("attachment")) { - pushPlaceholder(ct, "binary attachment", byteSizeFromBase64(bodyText)); - } else if (ct.startsWith("text/plain")) { - if (enc === "base64") { - parts.push(`[base64: ${byteSizeFromBase64(bodyText)} bytes]`); - } else { - parts.push(replaceInlineBase64(bodyText)); - } - } else if (ct.startsWith("text/html")) { - if (enc === "base64") { - parts.push(`[base64: ${byteSizeFromBase64(bodyText)} bytes]`); - } else { - let txt = parser.convertToPlainText(bodyText, - Ci.nsIDocumentEncoder.OutputLFLineBreak | - Ci.nsIDocumentEncoder.OutputNoScriptContent | - Ci.nsIDocumentEncoder.OutputNoFramesContent | - Ci.nsIDocumentEncoder.OutputBodyOnly, 0); - parts.push(replaceInlineBase64(txt)); - } - } else { - // Other single part types treated as attachments - pushPlaceholder(ct, "binary attachment", byteSizeFromBase64(bodyText)); - } - } - - walk(root); - return parts.join("\n"); - } catch (e) { - // Fallback: convert entire raw message to text - aiLog(`Failed to parse MIME, falling back to raw conversion`, {level: 'warn'}, e); - return parser.convertToPlainText(data, - Ci.nsIDocumentEncoder.OutputLFLineBreak | - Ci.nsIDocumentEncoder.OutputNoScriptContent | - Ci.nsIDocumentEncoder.OutputNoFramesContent | - Ci.nsIDocumentEncoder.OutputBodyOnly, 0); - } -} - From db767fae48de6cff2f4ba6b5d06de7f43f511787 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 02:46:49 -0500 Subject: [PATCH 020/124] Add maintenance tab and cache clearing --- README.md | 1 + modules/AiClassifier.js | 13 ++++++++++++- options/options.html | 12 ++++++++++++ options/options.js | 12 +++++++++++- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e6aecbe..3e60766 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ message meets a specified criterion. - **Context menu** – apply AI rules from the message list or the message display action button. - **Status icons** – toolbar icons show when classification is in progress and briefly display success or error states. - **Packaging script** – `build-xpi.ps1` builds an XPI ready for installation. +- **Maintenance tab** – view rule counts, cache entries and clear cached results from the options page. ### Cache Storage diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index a546fcd..de267b6 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -312,6 +312,17 @@ async function removeCacheEntries(keys = []) { } } +async function clearCache() { + if (!gCacheLoaded) { + await loadCache(); + } + if (gCache.size > 0) { + gCache.clear(); + await saveCache(); + aiLog(`[AiClassifier] Cleared cache`, {debug: true}); + } +} + function classifyTextSync(text, criterion, cacheKey = null) { if (!Services?.tm?.spinEventLoopUntil) { throw new Error("classifyTextSync requires Services"); @@ -396,4 +407,4 @@ async function init() { await loadCache(); } -export { classifyText, classifyTextSync, setConfig, removeCacheEntries, getReason, getCachedResult, buildCacheKey, buildCacheKeySync, init }; +export { classifyText, classifyTextSync, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, buildCacheKeySync, init }; diff --git a/options/options.html b/options/options.html index da9744d..f776fa2 100644 --- a/options/options.html +++ b/options/options.html @@ -46,6 +46,7 @@ @@ -171,6 +172,17 @@
+ + diff --git a/options/options.js b/options/options.js index 9bcf9dd..5536ad5 100644 --- a/options/options.js +++ b/options/options.js @@ -9,7 +9,8 @@ document.addEventListener('DOMContentLoaded', async () => { 'customSystemPrompt', 'aiParams', 'debugLogging', - 'aiRules' + 'aiRules', + 'aiCache' ]); const tabButtons = document.querySelectorAll('#main-tabs li'); const tabs = document.querySelectorAll('.tab-content'); @@ -300,6 +301,15 @@ document.addEventListener('DOMContentLoaded', async () => { if (r.stopProcessing) rule.stopProcessing = true; return rule; })); + + const ruleCountEl = document.getElementById('rule-count'); + const cacheCountEl = document.getElementById('cache-count'); + ruleCountEl.textContent = (defaults.aiRules || []).length; + cacheCountEl.textContent = defaults.aiCache ? Object.keys(defaults.aiCache).length : 0; + document.getElementById('clear-cache').addEventListener('click', async () => { + await AiClassifier.clearCache(); + cacheCountEl.textContent = '0'; + }); initialized = true; document.getElementById('save').addEventListener('click', async () => { From 8c03ad008e67833c7a2c954deaa2fce440541162 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 03:16:12 -0500 Subject: [PATCH 021/124] Add queue count stat to maintenance page --- background.js | 2 ++ options/options.html | 1 + options/options.js | 14 ++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/background.js b/background.js index b49e691..781b9df 100644 --- a/background.js +++ b/background.js @@ -388,6 +388,8 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("failed to clear cache for message", { level: 'error' }, e); return { ok: false }; } + } else if (msg?.type === "sortana:getQueueCount") { + return { count: queuedCount + (processing ? 1 : 0) }; } else { logger.aiLog("Unknown message type, ignoring", {level: 'warn'}, msg?.type); } diff --git a/options/options.html b/options/options.html index f776fa2..6ddefa7 100644 --- a/options/options.html +++ b/options/options.html @@ -179,6 +179,7 @@ Rule count Cache entries + Queue items diff --git a/options/options.js b/options/options.js index 5536ad5..7c1e595 100644 --- a/options/options.js +++ b/options/options.js @@ -304,8 +304,22 @@ document.addEventListener('DOMContentLoaded', async () => { const ruleCountEl = document.getElementById('rule-count'); const cacheCountEl = document.getElementById('cache-count'); + const queueCountEl = document.getElementById('queue-count'); ruleCountEl.textContent = (defaults.aiRules || []).length; cacheCountEl.textContent = defaults.aiCache ? Object.keys(defaults.aiCache).length : 0; + + async function refreshQueueCount() { + try { + const { count } = await browser.runtime.sendMessage({ type: 'sortana:getQueueCount' }); + queueCountEl.textContent = count; + } catch (e) { + queueCountEl.textContent = '?'; + } + } + + refreshQueueCount(); + setInterval(refreshQueueCount, 2000); + document.getElementById('clear-cache').addEventListener('click', async () => { await AiClassifier.clearCache(); cacheCountEl.textContent = '0'; From d5aecc6e8a8ede144c20269f64d3b81806eb6ea7 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 20:10:06 -0500 Subject: [PATCH 022/124] Add timing stats to Maintenance tab --- background.js | 35 +++++++++++++++++++++++++++++ modules/AiClassifier.js | 9 +++++++- options/options.html | 3 +++ options/options.js | 49 ++++++++++++++++++++++++++++++++++++----- 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/background.js b/background.js index 781b9df..6ef7eb5 100644 --- a/background.js +++ b/background.js @@ -19,6 +19,8 @@ let queue = Promise.resolve(); let queuedCount = 0; let processing = false; let iconTimer = null; +let timingStats = { count: 0, mean: 0, m2: 0, total: 0 }; +let currentStart = 0; function setIcon(path) { if (browser.browserAction) { @@ -106,6 +108,7 @@ async function applyAiRules(idsInput) { updateActionIcon(); queue = queue.then(async () => { processing = true; + currentStart = Date.now(); queuedCount--; updateActionIcon(); try { @@ -141,9 +144,27 @@ async function applyAiRules(idsInput) { } } processing = false; + const elapsed = Date.now() - currentStart; + currentStart = 0; + const t = timingStats; + t.count += 1; + t.total += elapsed; + const delta = elapsed - t.mean; + t.mean += delta / t.count; + t.m2 += delta * (elapsed - t.mean); + await storage.local.set({ classifyStats: t }); showTransientIcon("resources/img/done.png"); } catch (e) { processing = false; + const elapsed = Date.now() - currentStart; + currentStart = 0; + const t = timingStats; + t.count += 1; + t.total += elapsed; + const delta = elapsed - t.mean; + t.mean += delta / t.count; + t.m2 += delta * (elapsed - t.mean); + await storage.local.set({ classifyStats: t }); logger.aiLog("failed to apply AI rules", { level: 'error' }, e); showTransientIcon("resources/img/error.png"); } @@ -199,6 +220,10 @@ async function clearCacheForMessages(idsInput) { logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); await AiClassifier.init(); + const savedStats = await storage.local.get('classifyStats'); + if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { + Object.assign(timingStats, savedStats.classifyStats); + } aiRules = Array.isArray(store.aiRules) ? store.aiRules.map(r => { if (r.actions) return r; const actions = []; @@ -390,6 +415,16 @@ async function clearCacheForMessages(idsInput) { } } else if (msg?.type === "sortana:getQueueCount") { return { count: queuedCount + (processing ? 1 : 0) }; + } else if (msg?.type === "sortana:getTiming") { + const t = timingStats; + const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0; + return { + count: queuedCount + (processing ? 1 : 0), + current: currentStart ? Date.now() - currentStart : -1, + average: t.mean, + total: t.total, + stddev: std + }; } else { logger.aiLog("Unknown message type, ignoring", {level: 'warn'}, msg?.type); } diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index de267b6..3c526f8 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -323,6 +323,13 @@ async function clearCache() { } } +async function getCacheSize() { + if (!gCacheLoaded) { + await loadCache(); + } + return gCache.size; +} + function classifyTextSync(text, criterion, cacheKey = null) { if (!Services?.tm?.spinEventLoopUntil) { throw new Error("classifyTextSync requires Services"); @@ -407,4 +414,4 @@ async function init() { await loadCache(); } -export { classifyText, classifyTextSync, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, buildCacheKeySync, init }; +export { classifyText, classifyTextSync, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, buildCacheKeySync, getCacheSize, init }; diff --git a/options/options.html b/options/options.html index 6ddefa7..ed1ef41 100644 --- a/options/options.html +++ b/options/options.html @@ -180,6 +180,9 @@ Rule count Cache entries Queue items + Current run time--:-- + Average run time--:-- + Total run time--:-- diff --git a/options/options.js b/options/options.js index 7c1e595..997d7f7 100644 --- a/options/options.js +++ b/options/options.js @@ -305,20 +305,59 @@ document.addEventListener('DOMContentLoaded', async () => { const ruleCountEl = document.getElementById('rule-count'); const cacheCountEl = document.getElementById('cache-count'); const queueCountEl = document.getElementById('queue-count'); + const currentTimeEl = document.getElementById('current-time'); + const averageTimeEl = document.getElementById('average-time'); + const totalTimeEl = document.getElementById('total-time'); + let timingLogged = false; ruleCountEl.textContent = (defaults.aiRules || []).length; cacheCountEl.textContent = defaults.aiCache ? Object.keys(defaults.aiCache).length : 0; - async function refreshQueueCount() { + function format(ms) { + if (ms < 0) return '--:--'; + return (ms / 1000).toFixed(1) + 's'; + } + + async function refreshMaintenance() { try { - const { count } = await browser.runtime.sendMessage({ type: 'sortana:getQueueCount' }); - queueCountEl.textContent = count; + const stats = await browser.runtime.sendMessage({ type: 'sortana:getTiming' }); + queueCountEl.textContent = stats.count; + currentTimeEl.classList.remove('has-text-success','has-text-danger'); + let arrow = ''; + if (stats.current >= 0) { + if (stats.stddev > 0 && stats.current - stats.average > stats.stddev) { + currentTimeEl.classList.add('has-text-danger'); + arrow = ' ▲'; + } else if (stats.stddev > 0 && stats.average - stats.current > stats.stddev) { + currentTimeEl.classList.add('has-text-success'); + arrow = ' ▼'; + } + currentTimeEl.textContent = format(stats.current) + arrow; + } else { + currentTimeEl.textContent = '--:--'; + } + averageTimeEl.textContent = stats.count ? format(stats.average) : '--:--'; + totalTimeEl.textContent = format(stats.total); + if (!timingLogged) { + logger.aiLog('retrieved timing stats', {debug: true}); + timingLogged = true; + } } catch (e) { queueCountEl.textContent = '?'; + currentTimeEl.textContent = '--:--'; + averageTimeEl.textContent = '--:--'; + totalTimeEl.textContent = '--:--'; + } + + ruleCountEl.textContent = document.querySelectorAll('#rules-container .rule').length; + try { + cacheCountEl.textContent = await AiClassifier.getCacheSize(); + } catch { + cacheCountEl.textContent = '?'; } } - refreshQueueCount(); - setInterval(refreshQueueCount, 2000); + refreshMaintenance(); + setInterval(refreshMaintenance, 2000); document.getElementById('clear-cache').addEventListener('click', async () => { await AiClassifier.clearCache(); From 15de566068f62603bc367dede0750ca78a0ce7b3 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 20:10:38 -0500 Subject: [PATCH 023/124] Delete logger.jsm --- modules/logger.jsm | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 modules/logger.jsm diff --git a/modules/logger.jsm b/modules/logger.jsm deleted file mode 100644 index 7c64176..0000000 --- a/modules/logger.jsm +++ /dev/null @@ -1,26 +0,0 @@ -var EXPORTED_SYMBOLS = ['aiLog', 'setDebug']; -let debugEnabled = false; - -function setDebug(value) { - debugEnabled = !!value; -} - -function getCaller() { - try { - let stack = new Error().stack.split('\n'); - if (stack.length >= 3) { - return stack[2].trim().replace(/^@?\s*\(?/,'').replace(/^at\s+/, ''); - } - } catch (e) {} - return ''; -} - -function aiLog(message, opts = {}, ...args) { - const { level = 'log', debug = false } = opts; - if (debug && !debugEnabled) { - return; - } - const caller = getCaller(); - const prefix = caller ? `[ai-filter][${caller}]` : '[ai-filter]'; - console[level](`%c${prefix}`, 'color:#1c92d2;font-weight:bold', message, ...args); -} From c2e114266ebd929c4a4970fd3771b77f939e17fd Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 21:22:31 -0500 Subject: [PATCH 024/124] Update maintenance stats display --- background.js | 9 ++++++++- options/options.html | 7 ++++--- options/options.js | 44 ++++++++++++++++++++++++++++++-------------- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/background.js b/background.js index 6ef7eb5..3960e27 100644 --- a/background.js +++ b/background.js @@ -19,7 +19,7 @@ let queue = Promise.resolve(); let queuedCount = 0; let processing = false; let iconTimer = null; -let timingStats = { count: 0, mean: 0, m2: 0, total: 0 }; +let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 }; let currentStart = 0; function setIcon(path) { @@ -149,6 +149,7 @@ async function applyAiRules(idsInput) { const t = timingStats; t.count += 1; t.total += elapsed; + t.last = elapsed; const delta = elapsed - t.mean; t.mean += delta / t.count; t.m2 += delta * (elapsed - t.mean); @@ -161,6 +162,7 @@ async function applyAiRules(idsInput) { const t = timingStats; t.count += 1; t.total += elapsed; + t.last = elapsed; const delta = elapsed - t.mean; t.mean += delta / t.count; t.m2 += delta * (elapsed - t.mean); @@ -224,6 +226,9 @@ async function clearCacheForMessages(idsInput) { if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { Object.assign(timingStats, savedStats.classifyStats); } + if (typeof timingStats.last !== 'number') { + timingStats.last = -1; + } aiRules = Array.isArray(store.aiRules) ? store.aiRules.map(r => { if (r.actions) return r; const actions = []; @@ -421,6 +426,8 @@ async function clearCacheForMessages(idsInput) { return { count: queuedCount + (processing ? 1 : 0), current: currentStart ? Date.now() - currentStart : -1, + last: t.last, + runs: t.count, average: t.mean, total: t.total, stddev: std diff --git a/options/options.html b/options/options.html index ed1ef41..cb9668a 100644 --- a/options/options.html +++ b/options/options.html @@ -180,9 +180,10 @@ Rule count Cache entries Queue items - Current run time--:-- - Average run time--:-- - Total run time--:-- + Current run time--:--:-- + Last run time--:--:-- + Average run time--:--:-- + Total run time--:--:-- diff --git a/options/options.js b/options/options.js index 997d7f7..58dcd4f 100644 --- a/options/options.js +++ b/options/options.js @@ -306,6 +306,7 @@ document.addEventListener('DOMContentLoaded', async () => { const cacheCountEl = document.getElementById('cache-count'); const queueCountEl = document.getElementById('queue-count'); const currentTimeEl = document.getElementById('current-time'); + const lastTimeEl = document.getElementById('last-time'); const averageTimeEl = document.getElementById('average-time'); const totalTimeEl = document.getElementById('total-time'); let timingLogged = false; @@ -313,29 +314,43 @@ document.addEventListener('DOMContentLoaded', async () => { cacheCountEl.textContent = defaults.aiCache ? Object.keys(defaults.aiCache).length : 0; function format(ms) { - if (ms < 0) return '--:--'; - return (ms / 1000).toFixed(1) + 's'; + if (ms < 0) return '--:--:--'; + let totalSec = Math.floor(ms / 1000); + const sec = totalSec % 60; + totalSec = (totalSec - sec) / 60; + const min = totalSec % 60; + const hr = (totalSec - min) / 60; + return `${String(hr).padStart(2, '0')}:${String(min).padStart(2, '0')}:${String(sec).padStart(2, '0')}`; } async function refreshMaintenance() { try { const stats = await browser.runtime.sendMessage({ type: 'sortana:getTiming' }); queueCountEl.textContent = stats.count; - currentTimeEl.classList.remove('has-text-success','has-text-danger'); + currentTimeEl.classList.remove('has-text-danger'); + lastTimeEl.classList.remove('has-text-success','has-text-danger'); let arrow = ''; + if (stats.last >= 0) { + if (stats.stddev > 0 && stats.last - stats.average > stats.stddev) { + lastTimeEl.classList.add('has-text-danger'); + arrow = ' ▲'; + } else if (stats.stddev > 0 && stats.average - stats.last > stats.stddev) { + lastTimeEl.classList.add('has-text-success'); + arrow = ' ▼'; + } + lastTimeEl.textContent = format(stats.last) + arrow; + } else { + lastTimeEl.textContent = '--:--:--'; + } if (stats.current >= 0) { if (stats.stddev > 0 && stats.current - stats.average > stats.stddev) { currentTimeEl.classList.add('has-text-danger'); - arrow = ' ▲'; - } else if (stats.stddev > 0 && stats.average - stats.current > stats.stddev) { - currentTimeEl.classList.add('has-text-success'); - arrow = ' ▼'; } - currentTimeEl.textContent = format(stats.current) + arrow; + currentTimeEl.textContent = format(stats.current); } else { - currentTimeEl.textContent = '--:--'; + currentTimeEl.textContent = '--:--:--'; } - averageTimeEl.textContent = stats.count ? format(stats.average) : '--:--'; + averageTimeEl.textContent = stats.runs > 0 ? format(stats.average) : '--:--:--'; totalTimeEl.textContent = format(stats.total); if (!timingLogged) { logger.aiLog('retrieved timing stats', {debug: true}); @@ -343,9 +358,10 @@ document.addEventListener('DOMContentLoaded', async () => { } } catch (e) { queueCountEl.textContent = '?'; - currentTimeEl.textContent = '--:--'; - averageTimeEl.textContent = '--:--'; - totalTimeEl.textContent = '--:--'; + currentTimeEl.textContent = '--:--:--'; + lastTimeEl.textContent = '--:--:--'; + averageTimeEl.textContent = '--:--:--'; + totalTimeEl.textContent = '--:--:--'; } ruleCountEl.textContent = document.querySelectorAll('#rules-container .rule').length; @@ -357,7 +373,7 @@ document.addEventListener('DOMContentLoaded', async () => { } refreshMaintenance(); - setInterval(refreshMaintenance, 2000); + setInterval(refreshMaintenance, 1000); document.getElementById('clear-cache').addEventListener('click', async () => { await AiClassifier.clearCache(); From f9c1f0f048a063fe0537b541027041e73061ca2c Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 23:34:51 -0500 Subject: [PATCH 025/124] Update documentation for current code --- AGENTS.md | 6 +++--- README.md | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9ae810b..ab79c66 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,9 +5,9 @@ This file provides guidelines for codex agents contributing to the Sortana proje ## Repository Overview - `background.js`: Handles startup tasks and coordinates message passing within the extension. -- `modules/`: Holds reusable JavaScript modules for the extension. -- `content/`: Scripts for modifying Thunderbird windows (e.g., the filter editor). -- `options/`: The options page HTML and JavaScript. +- `modules/`: Contains reusable JavaScript modules such as `AiClassifier.js`. +- `options/`: The options page HTML, JavaScript and Bulma CSS. +- `details.html` and `details.js`: View AI reasoning and clear cache for a message. - `resources/`: Images and other static files. - `prompt_templates/`: Prompt template files for the AI service. - `build-xpi.ps1`: PowerShell script to package the extension. diff --git a/README.md b/README.md index 3e60766..ef4c429 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,10 @@ message meets a specified criterion. ## Features -- **AI classification rule** – adds the "AI classification" term with - `matches` and `doesn't match` operators. +- **AI classification rule** – adds the "AI classification" term with `matches` and `doesn't match` operators. - **Configurable endpoint** – set the classification service URL on the options page. - **Prompt templates** – choose between several model formats or provide your own custom template. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. -- **Filter editor integration** – patches Thunderbird's filter editor to accept - text criteria for AI classification. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. @@ -23,6 +20,9 @@ message meets a specified criterion. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. - **Context menu** – apply AI rules from the message list or the message display action button. - **Status icons** – toolbar icons show when classification is in progress and briefly display success or error states. +- **View reasoning** – inspect why rules matched via the Details popup. +- **Cache management** – clear cached results from the context menu or options page. +- **Queue & timing stats** – monitor processing time on the Maintenance tab. - **Packaging script** – `build-xpi.ps1` builds an XPI ready for installation. - **Maintenance tab** – view rule counts, cache entries and clear cached results from the options page. @@ -37,22 +37,22 @@ containing `matched` and `reason` fields. Older installations with a separate Sortana is implemented entirely with standard WebExtension scripts—no custom experiment code is required: -- `background.js` loads saved settings and listens for new messages. -- `modules/ExpressionSearchFilter.jsm` implements the AI filter and performs the - HTTP request. -- `options/` contains the HTML and JavaScript for configuring the endpoint and - rules. +- `background.js` loads saved settings, manages the classification queue and listens for new messages. +- `modules/AiClassifier.js` implements the classification logic and cache handling. +- `options/` contains the HTML and JavaScript for configuring the endpoint and rules. +- `details.html` / `details.js` present cached reasoning for a message. - `_locales/` holds localized strings used throughout the UI. ### Key Files | Path | Purpose | | --------------------------------------- | ---------------------------------------------- | -| `manifest.json` | Extension manifest and entry points. | -| `background.js` | Startup tasks and message handling. | -| `modules/ExpressionSearchFilter.jsm` | Custom filter term and AI request logic. | +| `manifest.json` | Extension manifest and entry points. | +| `background.js` | Startup tasks and classification queue management. | +| `modules/AiClassifier.js` | Core classification logic and cache handling. | | `options/options.html` and `options.js` | Endpoint and rule configuration UI. | -| `logger.js` and `modules/logger.jsm` | Colorized logging with optional debug mode. | +| `details.html` and `details.js` | View stored reasoning for a message. | +| `logger.js` | Colorized logging with optional debug mode. | ## Building From d7416c16ceb02f853d64fe44d5253acde525a3fe Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 29 Jun 2025 23:37:34 -0500 Subject: [PATCH 026/124] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ef4c429..2baa5e8 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ message meets a specified criterion. ## Features -- **AI classification rule** – adds the "AI classification" term with `matches` and `doesn't match` operators. - **Configurable endpoint** – set the classification service URL on the options page. - **Prompt templates** – choose between several model formats or provide your own custom template. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. From 86db83bb66103b2eb48363652cb17c1238ad5704 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 30 Jun 2025 01:56:22 -0500 Subject: [PATCH 027/124] Cleanup --- ai-filter.sln | 3 --- background.js | 12 ++++++++---- build-xpi.ps1 | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ai-filter.sln b/ai-filter.sln index 2d171dd..86ceaed 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -30,9 +30,6 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "modules", "modules", "{75ED3C1E-D3C7-4546-9F2E-AC85859DDF4B}" ProjectSection(SolutionItems) = preProject modules\AiClassifier.js = modules\AiClassifier.js - modules\ExpressionSearchFilter.jsm = modules\ExpressionSearchFilter.jsm - modules\logger.jsm = modules\logger.jsm - modules\messageUtils.jsm = modules\messageUtils.jsm EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_locales", "_locales", "{D446E5C6-BDDE-4091-BD1A-EC57170003CF}" diff --git a/background.js b/background.js index 3960e27..d0425c2 100644 --- a/background.js +++ b/background.js @@ -21,6 +21,7 @@ let processing = false; let iconTimer = null; let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 }; let currentStart = 0; +let logGetTiming = true; function setIcon(path) { if (browser.browserAction) { @@ -321,12 +322,15 @@ async function clearCacheForMessages(idsInput) { // Listen for messages from UI/devtools browser.runtime.onMessage.addListener(async (msg) => { - logger.aiLog("onMessage received", {debug: true}, msg); + if ((msg?.type === "sortana:getTiming" && logGetTiming) || (msg?.type !== "sortana:getTiming")) { + logGetTiming = false; + logger.aiLog("onMessage received", { debug: true }, msg); + } - if (msg?.type === "aiFilter:test") { + if (msg?.type === "sortana:test") { const { text = "", criterion = "" } = msg; - logger.aiLog("aiFilter:test – text", {debug: true}, text); - logger.aiLog("aiFilter:test – criterion", {debug: true}, criterion); + logger.aiLog("sortana:test – text", {debug: true}, text); + logger.aiLog("sortana:test – criterion", {debug: true}, criterion); try { logger.aiLog("Calling AiClassifier.classifyText()", {debug: true}); diff --git a/build-xpi.ps1 b/build-xpi.ps1 index 92a9474..708fe72 100644 --- a/build-xpi.ps1 +++ b/build-xpi.ps1 @@ -26,7 +26,7 @@ if (-not $version) { } # 4) Define output names & clean up -$xpiName = "ai-filter-$version.xpi" +$xpiName = "sortana-$version.xpi" $zipPath = Join-Path $ReleaseDir "ai-filter-$version.zip" $xpiPath = Join-Path $ReleaseDir $xpiName From 8ae32fe752d679e403e48d1cce0972e28cd712df Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Thu, 3 Jul 2025 03:07:59 -0500 Subject: [PATCH 028/124] Housekeeping for add-on store --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 3f17845..506e392 100644 --- a/manifest.json +++ b/manifest.json @@ -7,7 +7,7 @@ "gecko": { "id": "ai-filter@jordanwages", "strict_min_version": "128.0", - "strict_max_version": "*" + "strict_max_version": "140.*" } }, "icons": { From 1057d8c7fd77154d9e33a873bab5e6053998bbe7 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 01:51:32 -0500 Subject: [PATCH 029/124] Adding turndown for possible use Looking at adding html to markdown conversion for html email bodies. --- resources/js/turndown.js | 974 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 974 insertions(+) create mode 100644 resources/js/turndown.js diff --git a/resources/js/turndown.js b/resources/js/turndown.js new file mode 100644 index 0000000..e86fb18 --- /dev/null +++ b/resources/js/turndown.js @@ -0,0 +1,974 @@ +var TurndownService = (function () { + 'use strict'; + + function extend (destination) { + for (var i = 1; i < arguments.length; i++) { + var source = arguments[i]; + for (var key in source) { + if (source.hasOwnProperty(key)) destination[key] = source[key]; + } + } + return destination + } + + function repeat (character, count) { + return Array(count + 1).join(character) + } + + function trimLeadingNewlines (string) { + return string.replace(/^\n*/, '') + } + + function trimTrailingNewlines (string) { + // avoid match-at-end regexp bottleneck, see #370 + var indexEnd = string.length; + while (indexEnd > 0 && string[indexEnd - 1] === '\n') indexEnd--; + return string.substring(0, indexEnd) + } + + var blockElements = [ + 'ADDRESS', 'ARTICLE', 'ASIDE', 'AUDIO', 'BLOCKQUOTE', 'BODY', 'CANVAS', + 'CENTER', 'DD', 'DIR', 'DIV', 'DL', 'DT', 'FIELDSET', 'FIGCAPTION', 'FIGURE', + 'FOOTER', 'FORM', 'FRAMESET', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEADER', + 'HGROUP', 'HR', 'HTML', 'ISINDEX', 'LI', 'MAIN', 'MENU', 'NAV', 'NOFRAMES', + 'NOSCRIPT', 'OL', 'OUTPUT', 'P', 'PRE', 'SECTION', 'TABLE', 'TBODY', 'TD', + 'TFOOT', 'TH', 'THEAD', 'TR', 'UL' + ]; + + function isBlock (node) { + return is(node, blockElements) + } + + var voidElements = [ + 'AREA', 'BASE', 'BR', 'COL', 'COMMAND', 'EMBED', 'HR', 'IMG', 'INPUT', + 'KEYGEN', 'LINK', 'META', 'PARAM', 'SOURCE', 'TRACK', 'WBR' + ]; + + function isVoid (node) { + return is(node, voidElements) + } + + function hasVoid (node) { + return has(node, voidElements) + } + + var meaningfulWhenBlankElements = [ + 'A', 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TH', 'TD', 'IFRAME', 'SCRIPT', + 'AUDIO', 'VIDEO' + ]; + + function isMeaningfulWhenBlank (node) { + return is(node, meaningfulWhenBlankElements) + } + + function hasMeaningfulWhenBlank (node) { + return has(node, meaningfulWhenBlankElements) + } + + function is (node, tagNames) { + return tagNames.indexOf(node.nodeName) >= 0 + } + + function has (node, tagNames) { + return ( + node.getElementsByTagName && + tagNames.some(function (tagName) { + return node.getElementsByTagName(tagName).length + }) + ) + } + + var rules = {}; + + rules.paragraph = { + filter: 'p', + + replacement: function (content) { + return '\n\n' + content + '\n\n' + } + }; + + rules.lineBreak = { + filter: 'br', + + replacement: function (content, node, options) { + return options.br + '\n' + } + }; + + rules.heading = { + filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], + + replacement: function (content, node, options) { + var hLevel = Number(node.nodeName.charAt(1)); + + if (options.headingStyle === 'setext' && hLevel < 3) { + var underline = repeat((hLevel === 1 ? '=' : '-'), content.length); + return ( + '\n\n' + content + '\n' + underline + '\n\n' + ) + } else { + return '\n\n' + repeat('#', hLevel) + ' ' + content + '\n\n' + } + } + }; + + rules.blockquote = { + filter: 'blockquote', + + replacement: function (content) { + content = content.replace(/^\n+|\n+$/g, ''); + content = content.replace(/^/gm, '> '); + return '\n\n' + content + '\n\n' + } + }; + + rules.list = { + filter: ['ul', 'ol'], + + replacement: function (content, node) { + var parent = node.parentNode; + if (parent.nodeName === 'LI' && parent.lastElementChild === node) { + return '\n' + content + } else { + return '\n\n' + content + '\n\n' + } + } + }; + + rules.listItem = { + filter: 'li', + + replacement: function (content, node, options) { + content = content + .replace(/^\n+/, '') // remove leading newlines + .replace(/\n+$/, '\n') // replace trailing newlines with just a single one + .replace(/\n/gm, '\n '); // indent + var prefix = options.bulletListMarker + ' '; + var parent = node.parentNode; + if (parent.nodeName === 'OL') { + var start = parent.getAttribute('start'); + var index = Array.prototype.indexOf.call(parent.children, node); + prefix = (start ? Number(start) + index : index + 1) + '. '; + } + return ( + prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : '') + ) + } + }; + + rules.indentedCodeBlock = { + filter: function (node, options) { + return ( + options.codeBlockStyle === 'indented' && + node.nodeName === 'PRE' && + node.firstChild && + node.firstChild.nodeName === 'CODE' + ) + }, + + replacement: function (content, node, options) { + return ( + '\n\n ' + + node.firstChild.textContent.replace(/\n/g, '\n ') + + '\n\n' + ) + } + }; + + rules.fencedCodeBlock = { + filter: function (node, options) { + return ( + options.codeBlockStyle === 'fenced' && + node.nodeName === 'PRE' && + node.firstChild && + node.firstChild.nodeName === 'CODE' + ) + }, + + replacement: function (content, node, options) { + var className = node.firstChild.getAttribute('class') || ''; + var language = (className.match(/language-(\S+)/) || [null, ''])[1]; + var code = node.firstChild.textContent; + + var fenceChar = options.fence.charAt(0); + var fenceSize = 3; + var fenceInCodeRegex = new RegExp('^' + fenceChar + '{3,}', 'gm'); + + var match; + while ((match = fenceInCodeRegex.exec(code))) { + if (match[0].length >= fenceSize) { + fenceSize = match[0].length + 1; + } + } + + var fence = repeat(fenceChar, fenceSize); + + return ( + '\n\n' + fence + language + '\n' + + code.replace(/\n$/, '') + + '\n' + fence + '\n\n' + ) + } + }; + + rules.horizontalRule = { + filter: 'hr', + + replacement: function (content, node, options) { + return '\n\n' + options.hr + '\n\n' + } + }; + + rules.inlineLink = { + filter: function (node, options) { + return ( + options.linkStyle === 'inlined' && + node.nodeName === 'A' && + node.getAttribute('href') + ) + }, + + replacement: function (content, node) { + var href = node.getAttribute('href'); + if (href) href = href.replace(/([()])/g, '\\$1'); + var title = cleanAttribute(node.getAttribute('title')); + if (title) title = ' "' + title.replace(/"/g, '\\"') + '"'; + return '[' + content + '](' + href + title + ')' + } + }; + + rules.referenceLink = { + filter: function (node, options) { + return ( + options.linkStyle === 'referenced' && + node.nodeName === 'A' && + node.getAttribute('href') + ) + }, + + replacement: function (content, node, options) { + var href = node.getAttribute('href'); + var title = cleanAttribute(node.getAttribute('title')); + if (title) title = ' "' + title + '"'; + var replacement; + var reference; + + switch (options.linkReferenceStyle) { + case 'collapsed': + replacement = '[' + content + '][]'; + reference = '[' + content + ']: ' + href + title; + break + case 'shortcut': + replacement = '[' + content + ']'; + reference = '[' + content + ']: ' + href + title; + break + default: + var id = this.references.length + 1; + replacement = '[' + content + '][' + id + ']'; + reference = '[' + id + ']: ' + href + title; + } + + this.references.push(reference); + return replacement + }, + + references: [], + + append: function (options) { + var references = ''; + if (this.references.length) { + references = '\n\n' + this.references.join('\n') + '\n\n'; + this.references = []; // Reset references + } + return references + } + }; + + rules.emphasis = { + filter: ['em', 'i'], + + replacement: function (content, node, options) { + if (!content.trim()) return '' + return options.emDelimiter + content + options.emDelimiter + } + }; + + rules.strong = { + filter: ['strong', 'b'], + + replacement: function (content, node, options) { + if (!content.trim()) return '' + return options.strongDelimiter + content + options.strongDelimiter + } + }; + + rules.code = { + filter: function (node) { + var hasSiblings = node.previousSibling || node.nextSibling; + var isCodeBlock = node.parentNode.nodeName === 'PRE' && !hasSiblings; + + return node.nodeName === 'CODE' && !isCodeBlock + }, + + replacement: function (content) { + if (!content) return '' + content = content.replace(/\r?\n|\r/g, ' '); + + var extraSpace = /^`|^ .*?[^ ].* $|`$/.test(content) ? ' ' : ''; + var delimiter = '`'; + var matches = content.match(/`+/gm) || []; + while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`'; + + return delimiter + extraSpace + content + extraSpace + delimiter + } + }; + + rules.image = { + filter: 'img', + + replacement: function (content, node) { + var alt = cleanAttribute(node.getAttribute('alt')); + var src = node.getAttribute('src') || ''; + var title = cleanAttribute(node.getAttribute('title')); + var titlePart = title ? ' "' + title + '"' : ''; + return src ? '![' + alt + ']' + '(' + src + titlePart + ')' : '' + } + }; + + function cleanAttribute (attribute) { + return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : '' + } + + /** + * Manages a collection of rules used to convert HTML to Markdown + */ + + function Rules (options) { + this.options = options; + this._keep = []; + this._remove = []; + + this.blankRule = { + replacement: options.blankReplacement + }; + + this.keepReplacement = options.keepReplacement; + + this.defaultRule = { + replacement: options.defaultReplacement + }; + + this.array = []; + for (var key in options.rules) this.array.push(options.rules[key]); + } + + Rules.prototype = { + add: function (key, rule) { + this.array.unshift(rule); + }, + + keep: function (filter) { + this._keep.unshift({ + filter: filter, + replacement: this.keepReplacement + }); + }, + + remove: function (filter) { + this._remove.unshift({ + filter: filter, + replacement: function () { + return '' + } + }); + }, + + forNode: function (node) { + if (node.isBlank) return this.blankRule + var rule; + + if ((rule = findRule(this.array, node, this.options))) return rule + if ((rule = findRule(this._keep, node, this.options))) return rule + if ((rule = findRule(this._remove, node, this.options))) return rule + + return this.defaultRule + }, + + forEach: function (fn) { + for (var i = 0; i < this.array.length; i++) fn(this.array[i], i); + } + }; + + function findRule (rules, node, options) { + for (var i = 0; i < rules.length; i++) { + var rule = rules[i]; + if (filterValue(rule, node, options)) return rule + } + return void 0 + } + + function filterValue (rule, node, options) { + var filter = rule.filter; + if (typeof filter === 'string') { + if (filter === node.nodeName.toLowerCase()) return true + } else if (Array.isArray(filter)) { + if (filter.indexOf(node.nodeName.toLowerCase()) > -1) return true + } else if (typeof filter === 'function') { + if (filter.call(rule, node, options)) return true + } else { + throw new TypeError('`filter` needs to be a string, array, or function') + } + } + + /** + * The collapseWhitespace function is adapted from collapse-whitespace + * by Luc Thevenard. + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Luc Thevenard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + + /** + * collapseWhitespace(options) removes extraneous whitespace from an the given element. + * + * @param {Object} options + */ + function collapseWhitespace (options) { + var element = options.element; + var isBlock = options.isBlock; + var isVoid = options.isVoid; + var isPre = options.isPre || function (node) { + return node.nodeName === 'PRE' + }; + + if (!element.firstChild || isPre(element)) return + + var prevText = null; + var keepLeadingWs = false; + + var prev = null; + var node = next(prev, element, isPre); + + while (node !== element) { + if (node.nodeType === 3 || node.nodeType === 4) { // Node.TEXT_NODE or Node.CDATA_SECTION_NODE + var text = node.data.replace(/[ \r\n\t]+/g, ' '); + + if ((!prevText || / $/.test(prevText.data)) && + !keepLeadingWs && text[0] === ' ') { + text = text.substr(1); + } + + // `text` might be empty at this point. + if (!text) { + node = remove(node); + continue + } + + node.data = text; + + prevText = node; + } else if (node.nodeType === 1) { // Node.ELEMENT_NODE + if (isBlock(node) || node.nodeName === 'BR') { + if (prevText) { + prevText.data = prevText.data.replace(/ $/, ''); + } + + prevText = null; + keepLeadingWs = false; + } else if (isVoid(node) || isPre(node)) { + // Avoid trimming space around non-block, non-BR void elements and inline PRE. + prevText = null; + keepLeadingWs = true; + } else if (prevText) { + // Drop protection if set previously. + keepLeadingWs = false; + } + } else { + node = remove(node); + continue + } + + var nextNode = next(prev, node, isPre); + prev = node; + node = nextNode; + } + + if (prevText) { + prevText.data = prevText.data.replace(/ $/, ''); + if (!prevText.data) { + remove(prevText); + } + } + } + + /** + * remove(node) removes the given node from the DOM and returns the + * next node in the sequence. + * + * @param {Node} node + * @return {Node} node + */ + function remove (node) { + var next = node.nextSibling || node.parentNode; + + node.parentNode.removeChild(node); + + return next + } + + /** + * next(prev, current, isPre) returns the next node in the sequence, given the + * current and previous nodes. + * + * @param {Node} prev + * @param {Node} current + * @param {Function} isPre + * @return {Node} + */ + function next (prev, current, isPre) { + if ((prev && prev.parentNode === current) || isPre(current)) { + return current.nextSibling || current.parentNode + } + + return current.firstChild || current.nextSibling || current.parentNode + } + + /* + * Set up window for Node.js + */ + + var root = (typeof window !== 'undefined' ? window : {}); + + /* + * Parsing HTML strings + */ + + function canParseHTMLNatively () { + var Parser = root.DOMParser; + var canParse = false; + + // Adapted from https://gist.github.com/1129031 + // Firefox/Opera/IE throw errors on unsupported types + try { + // WebKit returns null on unsupported types + if (new Parser().parseFromString('', 'text/html')) { + canParse = true; + } + } catch (e) {} + + return canParse + } + + function createHTMLParser () { + var Parser = function () {}; + + { + if (shouldUseActiveX()) { + Parser.prototype.parseFromString = function (string) { + var doc = new window.ActiveXObject('htmlfile'); + doc.designMode = 'on'; // disable on-page scripts + doc.open(); + doc.write(string); + doc.close(); + return doc + }; + } else { + Parser.prototype.parseFromString = function (string) { + var doc = document.implementation.createHTMLDocument(''); + doc.open(); + doc.write(string); + doc.close(); + return doc + }; + } + } + return Parser + } + + function shouldUseActiveX () { + var useActiveX = false; + try { + document.implementation.createHTMLDocument('').open(); + } catch (e) { + if (root.ActiveXObject) useActiveX = true; + } + return useActiveX + } + + var HTMLParser = canParseHTMLNatively() ? root.DOMParser : createHTMLParser(); + + function RootNode (input, options) { + var root; + if (typeof input === 'string') { + var doc = htmlParser().parseFromString( + // DOM parsers arrange elements in the and . + // Wrapping in a custom element ensures elements are reliably arranged in + // a single element. + '' + input + '', + 'text/html' + ); + root = doc.getElementById('turndown-root'); + } else { + root = input.cloneNode(true); + } + collapseWhitespace({ + element: root, + isBlock: isBlock, + isVoid: isVoid, + isPre: options.preformattedCode ? isPreOrCode : null + }); + + return root + } + + var _htmlParser; + function htmlParser () { + _htmlParser = _htmlParser || new HTMLParser(); + return _htmlParser + } + + function isPreOrCode (node) { + return node.nodeName === 'PRE' || node.nodeName === 'CODE' + } + + function Node (node, options) { + node.isBlock = isBlock(node); + node.isCode = node.nodeName === 'CODE' || node.parentNode.isCode; + node.isBlank = isBlank(node); + node.flankingWhitespace = flankingWhitespace(node, options); + return node + } + + function isBlank (node) { + return ( + !isVoid(node) && + !isMeaningfulWhenBlank(node) && + /^\s*$/i.test(node.textContent) && + !hasVoid(node) && + !hasMeaningfulWhenBlank(node) + ) + } + + function flankingWhitespace (node, options) { + if (node.isBlock || (options.preformattedCode && node.isCode)) { + return { leading: '', trailing: '' } + } + + var edges = edgeWhitespace(node.textContent); + + // abandon leading ASCII WS if left-flanked by ASCII WS + if (edges.leadingAscii && isFlankedByWhitespace('left', node, options)) { + edges.leading = edges.leadingNonAscii; + } + + // abandon trailing ASCII WS if right-flanked by ASCII WS + if (edges.trailingAscii && isFlankedByWhitespace('right', node, options)) { + edges.trailing = edges.trailingNonAscii; + } + + return { leading: edges.leading, trailing: edges.trailing } + } + + function edgeWhitespace (string) { + var m = string.match(/^(([ \t\r\n]*)(\s*))(?:(?=\S)[\s\S]*\S)?((\s*?)([ \t\r\n]*))$/); + return { + leading: m[1], // whole string for whitespace-only strings + leadingAscii: m[2], + leadingNonAscii: m[3], + trailing: m[4], // empty for whitespace-only strings + trailingNonAscii: m[5], + trailingAscii: m[6] + } + } + + function isFlankedByWhitespace (side, node, options) { + var sibling; + var regExp; + var isFlanked; + + if (side === 'left') { + sibling = node.previousSibling; + regExp = / $/; + } else { + sibling = node.nextSibling; + regExp = /^ /; + } + + if (sibling) { + if (sibling.nodeType === 3) { + isFlanked = regExp.test(sibling.nodeValue); + } else if (options.preformattedCode && sibling.nodeName === 'CODE') { + isFlanked = false; + } else if (sibling.nodeType === 1 && !isBlock(sibling)) { + isFlanked = regExp.test(sibling.textContent); + } + } + return isFlanked + } + + var reduce = Array.prototype.reduce; + var escapes = [ + [/\\/g, '\\\\'], + [/\*/g, '\\*'], + [/^-/g, '\\-'], + [/^\+ /g, '\\+ '], + [/^(=+)/g, '\\$1'], + [/^(#{1,6}) /g, '\\$1 '], + [/`/g, '\\`'], + [/^~~~/g, '\\~~~'], + [/\[/g, '\\['], + [/\]/g, '\\]'], + [/^>/g, '\\>'], + [/_/g, '\\_'], + [/^(\d+)\. /g, '$1\\. '] + ]; + + function TurndownService (options) { + if (!(this instanceof TurndownService)) return new TurndownService(options) + + var defaults = { + rules: rules, + headingStyle: 'setext', + hr: '* * *', + bulletListMarker: '*', + codeBlockStyle: 'indented', + fence: '```', + emDelimiter: '_', + strongDelimiter: '**', + linkStyle: 'inlined', + linkReferenceStyle: 'full', + br: ' ', + preformattedCode: false, + blankReplacement: function (content, node) { + return node.isBlock ? '\n\n' : '' + }, + keepReplacement: function (content, node) { + return node.isBlock ? '\n\n' + node.outerHTML + '\n\n' : node.outerHTML + }, + defaultReplacement: function (content, node) { + return node.isBlock ? '\n\n' + content + '\n\n' : content + } + }; + this.options = extend({}, defaults, options); + this.rules = new Rules(this.options); + } + + TurndownService.prototype = { + /** + * The entry point for converting a string or DOM node to Markdown + * @public + * @param {String|HTMLElement} input The string or DOM node to convert + * @returns A Markdown representation of the input + * @type String + */ + + turndown: function (input) { + if (!canConvert(input)) { + throw new TypeError( + input + ' is not a string, or an element/document/fragment node.' + ) + } + + if (input === '') return '' + + var output = process.call(this, new RootNode(input, this.options)); + return postProcess.call(this, output) + }, + + /** + * Add one or more plugins + * @public + * @param {Function|Array} plugin The plugin or array of plugins to add + * @returns The Turndown instance for chaining + * @type Object + */ + + use: function (plugin) { + if (Array.isArray(plugin)) { + for (var i = 0; i < plugin.length; i++) this.use(plugin[i]); + } else if (typeof plugin === 'function') { + plugin(this); + } else { + throw new TypeError('plugin must be a Function or an Array of Functions') + } + return this + }, + + /** + * Adds a rule + * @public + * @param {String} key The unique key of the rule + * @param {Object} rule The rule + * @returns The Turndown instance for chaining + * @type Object + */ + + addRule: function (key, rule) { + this.rules.add(key, rule); + return this + }, + + /** + * Keep a node (as HTML) that matches the filter + * @public + * @param {String|Array|Function} filter The unique key of the rule + * @returns The Turndown instance for chaining + * @type Object + */ + + keep: function (filter) { + this.rules.keep(filter); + return this + }, + + /** + * Remove a node that matches the filter + * @public + * @param {String|Array|Function} filter The unique key of the rule + * @returns The Turndown instance for chaining + * @type Object + */ + + remove: function (filter) { + this.rules.remove(filter); + return this + }, + + /** + * Escapes Markdown syntax + * @public + * @param {String} string The string to escape + * @returns A string with Markdown syntax escaped + * @type String + */ + + escape: function (string) { + return escapes.reduce(function (accumulator, escape) { + return accumulator.replace(escape[0], escape[1]) + }, string) + } + }; + + /** + * Reduces a DOM node down to its Markdown string equivalent + * @private + * @param {HTMLElement} parentNode The node to convert + * @returns A Markdown representation of the node + * @type String + */ + + function process (parentNode) { + var self = this; + return reduce.call(parentNode.childNodes, function (output, node) { + node = new Node(node, self.options); + + var replacement = ''; + if (node.nodeType === 3) { + replacement = node.isCode ? node.nodeValue : self.escape(node.nodeValue); + } else if (node.nodeType === 1) { + replacement = replacementForNode.call(self, node); + } + + return join(output, replacement) + }, '') + } + + /** + * Appends strings as each rule requires and trims the output + * @private + * @param {String} output The conversion output + * @returns A trimmed version of the ouput + * @type String + */ + + function postProcess (output) { + var self = this; + this.rules.forEach(function (rule) { + if (typeof rule.append === 'function') { + output = join(output, rule.append(self.options)); + } + }); + + return output.replace(/^[\t\r\n]+/, '').replace(/[\t\r\n\s]+$/, '') + } + + /** + * Converts an element node to its Markdown equivalent + * @private + * @param {HTMLElement} node The node to convert + * @returns A Markdown representation of the node + * @type String + */ + + function replacementForNode (node) { + var rule = this.rules.forNode(node); + var content = process.call(this, node); + var whitespace = node.flankingWhitespace; + if (whitespace.leading || whitespace.trailing) content = content.trim(); + return ( + whitespace.leading + + rule.replacement(content, node, this.options) + + whitespace.trailing + ) + } + + /** + * Joins replacement to the current output with appropriate number of new lines + * @private + * @param {String} output The current conversion output + * @param {String} replacement The string to append to the output + * @returns Joined output + * @type String + */ + + function join (output, replacement) { + var s1 = trimTrailingNewlines(output); + var s2 = trimLeadingNewlines(replacement); + var nls = Math.max(output.length - s1.length, replacement.length - s2.length); + var separator = '\n\n'.substring(0, nls); + + return s1 + separator + s2 + } + + /** + * Determines whether an input can be converted + * @private + * @param {String|HTMLElement} input Describe this parameter + * @returns Describe what it returns + * @type String|Object|Array|Boolean|Number + */ + + function canConvert (input) { + return ( + input != null && ( + typeof input === 'string' || + (input.nodeType && ( + input.nodeType === 1 || input.nodeType === 9 || input.nodeType === 11 + )) + ) + ) + } + + return TurndownService; + +}()); From 5e0356086a0c4b527f461393ce4dea6cdb9a1e85 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 01:54:38 -0500 Subject: [PATCH 030/124] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2baa5e8..effdd3f 100644 --- a/README.md +++ b/README.md @@ -121,11 +121,14 @@ the disclosure is only required for add-on review, they'll be listed here as wel uses the following third party libraries: - [Bulma.css v1.0.4](https://github.com/jgthms/bulma/blob/1.0.4/css/bulma.css) + - MIT License +- [turndown v7.2.0](https://github.com/mixmark-io/turndown/tree/v7.2.0) + - MIT License ## License This project is licensed under the terms of the GNU General Public License -version 3. See `LICENSE` for the full text. +version 3. See `LICENSE` for the full text. Third party libraries are licensed seperately. ## Acknowledgments From 200c03c875fbcbe478730a4a492aa782def5fd95 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 02:26:58 -0500 Subject: [PATCH 031/124] Add Markdown conversion option --- README.md | 1 + _locales/en-US/messages.json | 1 + background.js | 23 +++++++++++++++++++++-- options/options.html | 5 +++++ options/options.js | 7 ++++++- resources/js/turndown.js | 3 +++ 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index effdd3f..a5fbce1 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ message meets a specified criterion. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. +- **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. - **Automatic rules** – create rules that tag or move new messages based on AI classification. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index 591373d..172a152 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -15,4 +15,5 @@ "template.custom": { "message": "Custom" }, "options.save": { "message": "Save" }, "options.debugLogging": { "message": "Enable debug logging" } + ,"options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" } } diff --git a/background.js b/background.js index d0425c2..2021990 100644 --- a/background.js +++ b/background.js @@ -22,6 +22,8 @@ let iconTimer = null; let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 }; let currentStart = 0; let logGetTiming = true; +let htmlToMarkdown = false; +let TurndownService = null; function setIcon(path) { if (browser.browserAction) { @@ -70,7 +72,17 @@ function collectText(part, bodyParts, attachments) { attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`); } else if (ct.startsWith("text/html")) { const doc = new DOMParser().parseFromString(body, 'text/html'); - bodyParts.push(replaceInlineBase64(doc.body.textContent || "")); + if (htmlToMarkdown && TurndownService) { + try { + const td = new TurndownService(); + const md = td.turndown(doc.body.innerHTML || body); + bodyParts.push(replaceInlineBase64(`[HTML Body converted to Markdown]\n${md}`)); + } catch (e) { + bodyParts.push(replaceInlineBase64(doc.body.textContent || "")); + } + } else { + bodyParts.push(replaceInlineBase64(doc.body.textContent || "")); + } } else { bodyParts.push(replaceInlineBase64(body)); } @@ -213,16 +225,19 @@ async function clearCacheForMessages(idsInput) { try { AiClassifier = await import(browser.runtime.getURL("modules/AiClassifier.js")); logger.aiLog("AiClassifier imported", {debug: true}); + const td = await import(browser.runtime.getURL("resources/js/turndown.js")); + TurndownService = td.default || td.TurndownService; } catch (e) { console.error("failed to import AiClassifier", e); return; } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "aiRules"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "aiRules"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); await AiClassifier.init(); + htmlToMarkdown = store.htmlToMarkdown === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { Object.assign(timingStats, savedStats.classifyStats); @@ -254,6 +269,10 @@ async function clearCacheForMessages(idsInput) { }); logger.aiLog("aiRules updated from storage change", {debug: true}, aiRules); } + if (changes.htmlToMarkdown) { + htmlToMarkdown = changes.htmlToMarkdown.newValue === true; + logger.aiLog("htmlToMarkdown updated from storage change", {debug: true}, htmlToMarkdown); + } }); } catch (err) { logger.aiLog("failed to load config", {level: 'error'}, err); diff --git a/options/options.html b/options/options.html index cb9668a..e285a22 100644 --- a/options/options.html +++ b/options/options.html @@ -98,6 +98,11 @@ Enable debug logging +
+ +
diff --git a/options/options.js b/options/options.js index 58dcd4f..1b8537a 100644 --- a/options/options.js +++ b/options/options.js @@ -9,6 +9,7 @@ document.addEventListener('DOMContentLoaded', async () => { 'customSystemPrompt', 'aiParams', 'debugLogging', + 'htmlToMarkdown', 'aiRules', 'aiCache' ]); @@ -81,6 +82,9 @@ document.addEventListener('DOMContentLoaded', async () => { const debugToggle = document.getElementById('debug-logging'); debugToggle.checked = defaults.debugLogging === true; + const htmlToggle = document.getElementById('html-to-markdown'); + htmlToggle.checked = defaults.htmlToMarkdown === true; + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { const el = document.getElementById(key); @@ -395,6 +399,7 @@ document.addEventListener('DOMContentLoaded', async () => { } } const debugLogging = debugToggle.checked; + const htmlToMarkdown = htmlToggle.checked; const rules = [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { const criterion = ruleEl.querySelector('.criterion').value; const actions = [...ruleEl.querySelectorAll('.action-row')].map(row => { @@ -413,7 +418,7 @@ document.addEventListener('DOMContentLoaded', async () => { const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; return { criterion, actions, stopProcessing }; }).filter(r => r.criterion); - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, aiRules: rules }); + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, aiRules: rules }); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); diff --git a/resources/js/turndown.js b/resources/js/turndown.js index e86fb18..cb9d04d 100644 --- a/resources/js/turndown.js +++ b/resources/js/turndown.js @@ -972,3 +972,6 @@ var TurndownService = (function () { return TurndownService; }()); + +export { TurndownService }; +export default TurndownService; From 149ff03cf902128281917712837344eed353501d Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 03:45:49 -0500 Subject: [PATCH 032/124] Add HTML sanitization options --- _locales/en-US/messages.json | 7 +++-- background.js | 61 ++++++++++++++++++++++++++++++++---- options/options.html | 15 +++++++++ options/options.js | 17 +++++++++- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index 172a152..97a356b 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -14,6 +14,9 @@ "template.mistral": { "message": "Mistral" }, "template.custom": { "message": "Custom" }, "options.save": { "message": "Save" }, - "options.debugLogging": { "message": "Enable debug logging" } - ,"options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" } + "options.debugLogging": { "message": "Enable debug logging" }, + "options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" }, + "options.stripUrlParams": { "message": "Remove URL tracking parameters" }, + "options.altTextImages": { "message": "Replace images with alt text" }, + "options.collapseWhitespace": { "message": "Collapse long whitespace" } } diff --git a/background.js b/background.js index 2021990..0613788 100644 --- a/background.js +++ b/background.js @@ -23,6 +23,9 @@ let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 }; let currentStart = 0; let logGetTiming = true; let htmlToMarkdown = false; +let stripUrlParams = false; +let altTextImages = false; +let collapseWhitespace = false; let TurndownService = null; function setIcon(path) { @@ -58,6 +61,20 @@ function replaceInlineBase64(text) { m => `[base64: ${byteSize(m)} bytes]`); } +function sanitizeString(text) { + let t = String(text); + if (stripUrlParams) { + t = t.replace(/https?:\/\/[^\s)]+/g, m => { + const idx = m.indexOf('?'); + return idx >= 0 ? m.slice(0, idx) : m; + }); + } + if (collapseWhitespace) { + t = t.replace(/[ \t\u00A0]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n'); + } + return t; +} + function collectText(part, bodyParts, attachments) { if (part.parts && part.parts.length) { for (const p of part.parts) collectText(p, bodyParts, attachments); @@ -72,19 +89,35 @@ function collectText(part, bodyParts, attachments) { attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`); } else if (ct.startsWith("text/html")) { const doc = new DOMParser().parseFromString(body, 'text/html'); + if (altTextImages) { + doc.querySelectorAll('img').forEach(img => { + const alt = img.getAttribute('alt') || ''; + img.replaceWith(doc.createTextNode(alt)); + }); + } + if (stripUrlParams) { + doc.querySelectorAll('[href]').forEach(a => { + const href = a.getAttribute('href'); + if (href) a.setAttribute('href', href.split('?')[0]); + }); + doc.querySelectorAll('[src]').forEach(e => { + const src = e.getAttribute('src'); + if (src) e.setAttribute('src', src.split('?')[0]); + }); + } if (htmlToMarkdown && TurndownService) { try { const td = new TurndownService(); - const md = td.turndown(doc.body.innerHTML || body); + const md = sanitizeString(td.turndown(doc.body.innerHTML || body)); bodyParts.push(replaceInlineBase64(`[HTML Body converted to Markdown]\n${md}`)); } catch (e) { - bodyParts.push(replaceInlineBase64(doc.body.textContent || "")); + bodyParts.push(replaceInlineBase64(sanitizeString(doc.body.textContent || ""))); } } else { - bodyParts.push(replaceInlineBase64(doc.body.textContent || "")); + bodyParts.push(replaceInlineBase64(sanitizeString(doc.body.textContent || ""))); } } else { - bodyParts.push(replaceInlineBase64(body)); + bodyParts.push(replaceInlineBase64(sanitizeString(body))); } } @@ -96,7 +129,8 @@ function buildEmailText(full) { .map(([k,v]) => `${k}: ${v.join(' ')}`) .join('\n'); const attachInfo = `Attachments: ${attachments.length}` + (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : ""); - return `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); + const combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); + return sanitizeString(combined); } async function applyAiRules(idsInput) { const ids = Array.isArray(idsInput) ? idsInput : [idsInput]; @@ -233,11 +267,14 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "aiRules"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "aiRules"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); await AiClassifier.init(); htmlToMarkdown = store.htmlToMarkdown === true; + stripUrlParams = store.stripUrlParams === true; + altTextImages = store.altTextImages === true; + collapseWhitespace = store.collapseWhitespace === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { Object.assign(timingStats, savedStats.classifyStats); @@ -273,6 +310,18 @@ async function clearCacheForMessages(idsInput) { htmlToMarkdown = changes.htmlToMarkdown.newValue === true; logger.aiLog("htmlToMarkdown updated from storage change", {debug: true}, htmlToMarkdown); } + if (changes.stripUrlParams) { + stripUrlParams = changes.stripUrlParams.newValue === true; + logger.aiLog("stripUrlParams updated from storage change", {debug: true}, stripUrlParams); + } + if (changes.altTextImages) { + altTextImages = changes.altTextImages.newValue === true; + logger.aiLog("altTextImages updated from storage change", {debug: true}, altTextImages); + } + if (changes.collapseWhitespace) { + collapseWhitespace = changes.collapseWhitespace.newValue === true; + logger.aiLog("collapseWhitespace updated from storage change", {debug: true}, collapseWhitespace); + } }); } catch (err) { logger.aiLog("failed to load config", {level: 'error'}, err); diff --git a/options/options.html b/options/options.html index e285a22..f40cda2 100644 --- a/options/options.html +++ b/options/options.html @@ -103,6 +103,21 @@ Convert HTML body to Markdown
+
+ +
+
+ +
+
+ +
diff --git a/options/options.js b/options/options.js index 1b8537a..2350efb 100644 --- a/options/options.js +++ b/options/options.js @@ -10,6 +10,9 @@ document.addEventListener('DOMContentLoaded', async () => { 'aiParams', 'debugLogging', 'htmlToMarkdown', + 'stripUrlParams', + 'altTextImages', + 'collapseWhitespace', 'aiRules', 'aiCache' ]); @@ -85,6 +88,15 @@ document.addEventListener('DOMContentLoaded', async () => { const htmlToggle = document.getElementById('html-to-markdown'); htmlToggle.checked = defaults.htmlToMarkdown === true; + const stripUrlToggle = document.getElementById('strip-url-params'); + stripUrlToggle.checked = defaults.stripUrlParams === true; + + const altTextToggle = document.getElementById('alt-text-images'); + altTextToggle.checked = defaults.altTextImages === true; + + const collapseWhitespaceToggle = document.getElementById('collapse-whitespace'); + collapseWhitespaceToggle.checked = defaults.collapseWhitespace === true; + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { const el = document.getElementById(key); @@ -418,7 +430,10 @@ document.addEventListener('DOMContentLoaded', async () => { const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; return { criterion, actions, stopProcessing }; }).filter(r => r.criterion); - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, aiRules: rules }); + const stripUrlParams = stripUrlToggle.checked; + const altTextImages = altTextToggle.checked; + const collapseWhitespace = collapseWhitespaceToggle.checked; + await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, aiRules: rules }); try { await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); From 41769c0e96384f9cbf6896923e2dc249e5759ca3 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 04:33:01 -0500 Subject: [PATCH 033/124] Handle preview pane when loading details --- README.md | 1 + details.js | 6 ++++++ manifest.json | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a5fbce1..b9ee4be 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Sortana requests the following Thunderbird permissions: - `messagesTagsList` – retrieve existing message tags for rule actions. - `accountsRead` – list accounts and folders for move actions. - `menus` – add context menu commands. +- `tabs` – open new tabs and query the active tab. ## Thunderbird Add-on Store Disclosures diff --git a/details.js b/details.js index 0d190d5..4442ab1 100644 --- a/details.js +++ b/details.js @@ -8,6 +8,12 @@ document.addEventListener('DOMContentLoaded', async () => { const tabId = tabs[0]?.id; const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; id = msgs[0]?.id; + if (!id) { + const mailTabs = await browser.mailTabs.query({ active: true, currentWindow: true }); + const mailTabId = mailTabs[0]?.id; + const selected = mailTabId !== undefined ? await browser.mailTabs.getSelectedMessages(mailTabId) : null; + id = selected?.messages?.[0]?.id; + } } catch (e) { console.error('failed to determine message id', e); } diff --git a/manifest.json b/manifest.json index 506e392..879d1f3 100644 --- a/manifest.json +++ b/manifest.json @@ -40,6 +40,7 @@ "messagesTagsList", "accountsRead", "menus", - "scripting" + "scripting", + "tabs" ] } From c0ba2d1fdd707423eb68df99e1f032c376dfb221 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 04:50:18 -0500 Subject: [PATCH 034/124] Add logging to details page --- details.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/details.js b/details.js index 4442ab1..1985aa9 100644 --- a/details.js +++ b/details.js @@ -1,6 +1,13 @@ document.addEventListener('DOMContentLoaded', async () => { + const storage = (globalThis.messenger ?? browser).storage; + const logger = await import(browser.runtime.getURL('logger.js')); + const { debugLogging } = await storage.local.get('debugLogging'); + logger.setDebug(debugLogging === true); + logger.aiLog('details page loaded', { debug: true }); + const params = new URLSearchParams(location.search); let id = parseInt(params.get('mid'), 10); + logger.aiLog('initial message id', { debug: true }, id); if (!id) { try { @@ -8,22 +15,27 @@ document.addEventListener('DOMContentLoaded', async () => { const tabId = tabs[0]?.id; const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; id = msgs[0]?.id; + logger.aiLog('message id from displayed messages', { debug: true }, id); if (!id) { const mailTabs = await browser.mailTabs.query({ active: true, currentWindow: true }); const mailTabId = mailTabs[0]?.id; const selected = mailTabId !== undefined ? await browser.mailTabs.getSelectedMessages(mailTabId) : null; id = selected?.messages?.[0]?.id; + logger.aiLog('message id from selected messages', { debug: true }, id); } } catch (e) { - console.error('failed to determine message id', e); + logger.aiLog('failed to determine message id', { level: 'error' }, e); } } if (!id) return; try { + logger.aiLog('requesting message details', {}, id); const { subject, results } = await browser.runtime.sendMessage({ type: 'sortana:getDetails', id }); + logger.aiLog('received details', { debug: true }, { subject, results }); document.getElementById('subject').textContent = subject; const container = document.getElementById('rules'); for (const r of results) { + logger.aiLog('rendering rule result', { debug: true }, r); const article = document.createElement('article'); const color = r.matched === true ? 'is-success' : 'is-danger'; article.className = `message ${color} mb-4`; @@ -43,10 +55,11 @@ document.addEventListener('DOMContentLoaded', async () => { container.appendChild(article); } document.getElementById('clear').addEventListener('click', async () => { + logger.aiLog('clearing cache for message', {}, id); await browser.runtime.sendMessage({ type: 'sortana:clearCacheForMessage', id }); window.close(); }); } catch (e) { - console.error('failed to load details', e); + logger.aiLog('failed to load details', { level: 'error' }, e); } }); From 6b741595cced2d29f24d04418cdaf7b45b61eb1f Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 05:07:48 -0500 Subject: [PATCH 035/124] Fix message lookup in details popup --- background.js | 2 +- details.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/background.js b/background.js index 0613788..bc3c5dd 100644 --- a/background.js +++ b/background.js @@ -413,7 +413,7 @@ async function clearCacheForMessages(idsInput) { } } else if (msg?.type === "sortana:clearCacheForDisplayed") { try { - const tabs = await browser.tabs.query({ active: true, currentWindow: true }); + const tabs = await browser.tabs.query({ active: true, lastFocusedWindow: true }); const tabId = tabs[0]?.id; const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; const ids = msgs.map(m => m.id); diff --git a/details.js b/details.js index 1985aa9..c586cc2 100644 --- a/details.js +++ b/details.js @@ -11,13 +11,13 @@ document.addEventListener('DOMContentLoaded', async () => { if (!id) { try { - const tabs = await browser.tabs.query({ active: true, currentWindow: true }); + const tabs = await browser.tabs.query({ active: true, lastFocusedWindow: true }); const tabId = tabs[0]?.id; const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; id = msgs[0]?.id; logger.aiLog('message id from displayed messages', { debug: true }, id); if (!id) { - const mailTabs = await browser.mailTabs.query({ active: true, currentWindow: true }); + const mailTabs = await browser.mailTabs.query({ active: true, lastFocusedWindow: true }); const mailTabId = mailTabs[0]?.id; const selected = mailTabId !== undefined ? await browser.mailTabs.getSelectedMessages(mailTabId) : null; id = selected?.messages?.[0]?.id; From 9724c19b7d1a7518f46c8b08bcda9595c33d1118 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 19:33:05 -0500 Subject: [PATCH 036/124] Debug attempt --- ai-filter.sln | 6 ++++++ details.js | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ai-filter.sln b/ai-filter.sln index 86ceaed..7922392 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -64,6 +64,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-175 resources\img\logo96.png = resources\img\logo96.png EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "js", "js", "{21D2A42C-3F85-465C-9141-C106AFD92B68}" + ProjectSection(SolutionItems) = preProject + resources\js\turndown.js = resources\js\turndown.js + EndProjectSection +EndProject Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -76,5 +81,6 @@ Global {86516D53-50D4-4FE2-9D8A-977A8F5EBDBD} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {68A87938-5C2B-49F5-8AAA-8A34FBBFD854} = {BCC6E6D2-343B-4C48-854D-5FE3BBC3CB70} {F266602F-1755-4A95-A11B-6C90C701C5BF} = {68A87938-5C2B-49F5-8AAA-8A34FBBFD854} + {21D2A42C-3F85-465C-9141-C106AFD92B68} = {68A87938-5C2B-49F5-8AAA-8A34FBBFD854} EndGlobalSection EndGlobal diff --git a/details.js b/details.js index c586cc2..28c7504 100644 --- a/details.js +++ b/details.js @@ -11,7 +11,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (!id) { try { - const tabs = await browser.tabs.query({ active: true, lastFocusedWindow: true }); + const tabs = await messenger.tabs.query({ active: true, currentWindow: true }); const tabId = tabs[0]?.id; const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; id = msgs[0]?.id; From 79f49fd5028314b7740d7eac6feb609b88e0fbe5 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 19:58:11 -0500 Subject: [PATCH 037/124] Simplify message lookup --- background.js | 4 +--- details.js | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/background.js b/background.js index bc3c5dd..7198e23 100644 --- a/background.js +++ b/background.js @@ -413,9 +413,7 @@ async function clearCacheForMessages(idsInput) { } } else if (msg?.type === "sortana:clearCacheForDisplayed") { try { - const tabs = await browser.tabs.query({ active: true, lastFocusedWindow: true }); - const tabId = tabs[0]?.id; - const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; + const msgs = await browser.messageDisplay.getDisplayedMessages(); const ids = msgs.map(m => m.id); await clearCacheForMessages(ids); } catch (e) { diff --git a/details.js b/details.js index c586cc2..6ea15fe 100644 --- a/details.js +++ b/details.js @@ -11,15 +11,11 @@ document.addEventListener('DOMContentLoaded', async () => { if (!id) { try { - const tabs = await browser.tabs.query({ active: true, lastFocusedWindow: true }); - const tabId = tabs[0]?.id; - const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; + const msgs = await browser.messageDisplay.getDisplayedMessages(); id = msgs[0]?.id; logger.aiLog('message id from displayed messages', { debug: true }, id); if (!id) { - const mailTabs = await browser.mailTabs.query({ active: true, lastFocusedWindow: true }); - const mailTabId = mailTabs[0]?.id; - const selected = mailTabId !== undefined ? await browser.mailTabs.getSelectedMessages(mailTabId) : null; + const selected = await browser.mailTabs.getSelectedMessages(); id = selected?.messages?.[0]?.id; logger.aiLog('message id from selected messages', { debug: true }, id); } From 846d1270c55bf511132c9504b7013531545dc181 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 20:02:51 -0500 Subject: [PATCH 038/124] Updating version number since we're making significant changes --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 879d1f3..36b94b0 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.0.0", + "version": "2.1.0", "default_locale": "en-US", "applications": { "gecko": { From 97bfabfbea8aafa17db6f573954c6901df3b1469 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 5 Jul 2025 22:53:40 -0500 Subject: [PATCH 039/124] Add fallback message to fetch active message id --- background.js | 13 +++++++++++++ details.js | 13 ++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/background.js b/background.js index 7198e23..58852c2 100644 --- a/background.js +++ b/background.js @@ -411,6 +411,19 @@ async function clearCacheForMessages(idsInput) { // rethrow so the caller sees the failure throw err; } + } else if (msg?.type === "sortana:getActiveMessage") { + try { + const displayed = await browser.messageDisplay.getDisplayedMessages(); + let id = displayed[0]?.id; + if (!id) { + const selected = await browser.mailTabs.getSelectedMessages(); + id = selected?.messages?.[0]?.id; + } + return { id: id ?? null }; + } catch (e) { + logger.aiLog("failed to get active message", { level: 'error' }, e); + return { id: null }; + } } else if (msg?.type === "sortana:clearCacheForDisplayed") { try { const msgs = await browser.messageDisplay.getDisplayedMessages(); diff --git a/details.js b/details.js index 6ea15fe..3b78e8a 100644 --- a/details.js +++ b/details.js @@ -20,9 +20,20 @@ document.addEventListener('DOMContentLoaded', async () => { logger.aiLog('message id from selected messages', { debug: true }, id); } } catch (e) { - logger.aiLog('failed to determine message id', { level: 'error' }, e); + logger.aiLog('failed to determine message id locally', { level: 'error' }, e); } } + + if (!id) { + try { + const resp = await browser.runtime.sendMessage({ type: 'sortana:getActiveMessage' }); + id = resp?.id; + logger.aiLog('message id from background', { debug: true }, id); + } catch (e) { + logger.aiLog('failed to get message id from background', { level: 'error' }, e); + } + } + if (!id) return; try { logger.aiLog('requesting message details', {}, id); From eb91474f5a0e6b44f8ee803a7ade66b53a0ee8be Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 00:05:31 -0500 Subject: [PATCH 040/124] Migrate manifest to version 3 --- manifest.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifest.json b/manifest.json index 36b94b0..8d1ab76 100644 --- a/manifest.json +++ b/manifest.json @@ -1,9 +1,9 @@ { - "manifest_version": 2, + "manifest_version": 3, "name": "Sortana", "version": "2.1.0", "default_locale": "en-US", - "applications": { + "browser_specific_settings": { "gecko": { "id": "ai-filter@jordanwages", "strict_min_version": "128.0", @@ -18,7 +18,7 @@ "96": "resources/img/logo96.png", "128": "resources/img/logo128.png" }, - "browser_action": { + "action": { "default_icon": "resources/img/logo32.png" }, "message_display_action": { From 8f5165dcec4a12b50a9f33fe61f89cd53605b620 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 00:29:46 -0500 Subject: [PATCH 041/124] Add host permissions for endpoint access --- README.md | 1 + manifest.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index b9ee4be..eff53b1 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ Sortana requests the following Thunderbird permissions: - `accountsRead` – list accounts and folders for move actions. - `menus` – add context menu commands. - `tabs` – open new tabs and query the active tab. +- Host permissions (`*://*/*`) – allow network requests to your configured classification service. ## Thunderbird Add-on Store Disclosures diff --git a/manifest.json b/manifest.json index 8d1ab76..d43df2c 100644 --- a/manifest.json +++ b/manifest.json @@ -42,5 +42,8 @@ "menus", "scripting", "tabs" + ], + "host_permissions": [ + "*://*/*" ] } From 0c07479fa987cfa96ed1a1ddcbd05492841e26ec Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 01:22:44 -0500 Subject: [PATCH 042/124] Added CSP --- manifest.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index d43df2c..2ff6b74 100644 --- a/manifest.json +++ b/manifest.json @@ -45,5 +45,8 @@ ], "host_permissions": [ "*://*/*" - ] + ], + "content_security_policy": { + "extension_pages": "script-src 'self'; object-src 'none'; connect-src 'self' http: https:" + } } From d60725eb4b3fb20755a555f0edad927f91eb9b67 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 02:10:46 -0500 Subject: [PATCH 043/124] Handle message display in TB 128 --- background.js | 39 ++++++++++++++-------------- details.js | 70 +++++++++++++++++++++++---------------------------- manifest.json | 3 +-- 3 files changed, 52 insertions(+), 60 deletions(-) diff --git a/background.js b/background.js index 58852c2..82c1fd0 100644 --- a/background.js +++ b/background.js @@ -333,6 +333,19 @@ async function clearCacheForMessages(idsInput) { if (browser.messageDisplayAction.setLabel) { browser.messageDisplayAction.setLabel({ label: "Details" }); } + + browser.messageDisplayAction.onClicked.addListener(async (tab) => { + const header = await browser.messageDisplay.getDisplayedMessage(tab.id); + if (!header) { + console.warn("[Sortana] no displayed message in tab", tab.id); + return; + } + + const popupUrl = `${browser.runtime.getURL("details.html")}?mid=${header.id}`; + + await browser.messageDisplayAction.setPopup({ tabId: tab.id, popup: popupUrl }); + await browser.messageDisplayAction.openPopup({ tabId: tab.id }); + }); } browser.menus.create({ @@ -370,7 +383,7 @@ async function clearCacheForMessages(idsInput) { - browser.menus.onClicked.addListener(async info => { + browser.menus.onClicked.addListener(async (info, tab) => { if (info.menuItemId === "apply-ai-rules-list" || info.menuItemId === "apply-ai-rules-display") { const ids = info.selectedMessages?.messages?.map(m => m.id) || (info.messageId ? [info.messageId] : []); @@ -380,11 +393,12 @@ async function clearCacheForMessages(idsInput) { (info.messageId ? [info.messageId] : []); await clearCacheForMessages(ids); } else if (info.menuItemId === "view-ai-reason-list" || info.menuItemId === "view-ai-reason-display") { - const id = info.messageId || info.selectedMessages?.messages?.[0]?.id; - if (id) { - const url = browser.runtime.getURL(`details.html?mid=${id}`); - browser.tabs.create({ url }); - } + const header = await browser.messageDisplay.getDisplayedMessage(tab.id); + if (!header) { return; } + + const url = `${browser.runtime.getURL("details.html")}?mid=${header.id}`; + + await browser.tabs.create({ url }); } }); @@ -411,19 +425,6 @@ async function clearCacheForMessages(idsInput) { // rethrow so the caller sees the failure throw err; } - } else if (msg?.type === "sortana:getActiveMessage") { - try { - const displayed = await browser.messageDisplay.getDisplayedMessages(); - let id = displayed[0]?.id; - if (!id) { - const selected = await browser.mailTabs.getSelectedMessages(); - id = selected?.messages?.[0]?.id; - } - return { id: id ?? null }; - } catch (e) { - logger.aiLog("failed to get active message", { level: 'error' }, e); - return { id: null }; - } } else if (msg?.type === "sortana:clearCacheForDisplayed") { try { const msgs = await browser.messageDisplay.getDisplayedMessages(); diff --git a/details.js b/details.js index 3b78e8a..c5488f6 100644 --- a/details.js +++ b/details.js @@ -1,48 +1,40 @@ document.addEventListener('DOMContentLoaded', async () => { + const logger = (await import(browser.runtime.getURL('logger.js'))).aiLog; + + const midParam = new URLSearchParams(location.search).get('mid'); + const messageId = parseInt(midParam, 10); + + if (!messageId) { + logger('no ?mid → trying displayedMessage fallback'); + const openerTabId = (await browser.tabs.getCurrent()).openerTabId; + const header = await browser.messageDisplay.getDisplayedMessage(openerTabId); + if (!header) { + logger('still no message – aborting'); + return; + } + loadMessage(header.id); + return; + } + + loadMessage(messageId); +}); + +async function loadMessage(id) { const storage = (globalThis.messenger ?? browser).storage; - const logger = await import(browser.runtime.getURL('logger.js')); + const logMod = await import(browser.runtime.getURL('logger.js')); const { debugLogging } = await storage.local.get('debugLogging'); - logger.setDebug(debugLogging === true); - logger.aiLog('details page loaded', { debug: true }); + logMod.setDebug(debugLogging === true); + const log = logMod.aiLog; - const params = new URLSearchParams(location.search); - let id = parseInt(params.get('mid'), 10); - logger.aiLog('initial message id', { debug: true }, id); - - if (!id) { - try { - const msgs = await browser.messageDisplay.getDisplayedMessages(); - id = msgs[0]?.id; - logger.aiLog('message id from displayed messages', { debug: true }, id); - if (!id) { - const selected = await browser.mailTabs.getSelectedMessages(); - id = selected?.messages?.[0]?.id; - logger.aiLog('message id from selected messages', { debug: true }, id); - } - } catch (e) { - logger.aiLog('failed to determine message id locally', { level: 'error' }, e); - } - } - - if (!id) { - try { - const resp = await browser.runtime.sendMessage({ type: 'sortana:getActiveMessage' }); - id = resp?.id; - logger.aiLog('message id from background', { debug: true }, id); - } catch (e) { - logger.aiLog('failed to get message id from background', { level: 'error' }, e); - } - } - - if (!id) return; + log('details page loaded', { debug: true }); try { - logger.aiLog('requesting message details', {}, id); + log('requesting message details', {}, id); const { subject, results } = await browser.runtime.sendMessage({ type: 'sortana:getDetails', id }); - logger.aiLog('received details', { debug: true }, { subject, results }); + log('received details', { debug: true }, { subject, results }); document.getElementById('subject').textContent = subject; const container = document.getElementById('rules'); for (const r of results) { - logger.aiLog('rendering rule result', { debug: true }, r); + log('rendering rule result', { debug: true }, r); const article = document.createElement('article'); const color = r.matched === true ? 'is-success' : 'is-danger'; article.className = `message ${color} mb-4`; @@ -62,11 +54,11 @@ document.addEventListener('DOMContentLoaded', async () => { container.appendChild(article); } document.getElementById('clear').addEventListener('click', async () => { - logger.aiLog('clearing cache for message', {}, id); + log('clearing cache for message', {}, id); await browser.runtime.sendMessage({ type: 'sortana:clearCacheForMessage', id }); window.close(); }); } catch (e) { - logger.aiLog('failed to load details', { level: 'error' }, e); + log('failed to load details', { level: 'error' }, e); } -}); +} diff --git a/manifest.json b/manifest.json index 2ff6b74..77506b2 100644 --- a/manifest.json +++ b/manifest.json @@ -24,8 +24,7 @@ "message_display_action": { "default_icon": "resources/img/brain.png", "default_title": "Details", - "default_label": "Details", - "default_popup": "details.html" + "default_label": "Details" }, "background": { "scripts": [ "background.js" ] }, "options_ui": { From 34cf8e234e87ce6750a774be7acc5c9f558da21a Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 02:36:42 -0500 Subject: [PATCH 044/124] Restore popup defaults and update message lookups --- background.js | 14 +------------- details.js | 29 ++++++++++++++--------------- manifest.json | 3 ++- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/background.js b/background.js index 82c1fd0..6d0b4e8 100644 --- a/background.js +++ b/background.js @@ -334,18 +334,6 @@ async function clearCacheForMessages(idsInput) { browser.messageDisplayAction.setLabel({ label: "Details" }); } - browser.messageDisplayAction.onClicked.addListener(async (tab) => { - const header = await browser.messageDisplay.getDisplayedMessage(tab.id); - if (!header) { - console.warn("[Sortana] no displayed message in tab", tab.id); - return; - } - - const popupUrl = `${browser.runtime.getURL("details.html")}?mid=${header.id}`; - - await browser.messageDisplayAction.setPopup({ tabId: tab.id, popup: popupUrl }); - await browser.messageDisplayAction.openPopup({ tabId: tab.id }); - }); } browser.menus.create({ @@ -393,7 +381,7 @@ async function clearCacheForMessages(idsInput) { (info.messageId ? [info.messageId] : []); await clearCacheForMessages(ids); } else if (info.menuItemId === "view-ai-reason-list" || info.menuItemId === "view-ai-reason-display") { - const header = await browser.messageDisplay.getDisplayedMessage(tab.id); + const [header] = await browser.messageDisplay.getDisplayedMessages(tab.id); if (!header) { return; } const url = `${browser.runtime.getURL("details.html")}?mid=${header.id}`; diff --git a/details.js b/details.js index c5488f6..016783a 100644 --- a/details.js +++ b/details.js @@ -1,22 +1,21 @@ -document.addEventListener('DOMContentLoaded', async () => { - const logger = (await import(browser.runtime.getURL('logger.js'))).aiLog; +document.addEventListener("DOMContentLoaded", async () => { + const aiLog = (await import(browser.runtime.getURL("logger.js"))).aiLog; - const midParam = new URLSearchParams(location.search).get('mid'); - const messageId = parseInt(midParam, 10); - - if (!messageId) { - logger('no ?mid → trying displayedMessage fallback'); - const openerTabId = (await browser.tabs.getCurrent()).openerTabId; - const header = await browser.messageDisplay.getDisplayedMessage(openerTabId); - if (!header) { - logger('still no message – aborting'); - return; - } - loadMessage(header.id); + const qMid = parseInt(new URLSearchParams(location.search).get("mid"), 10); + if (!isNaN(qMid)) { + loadMessage(qMid); return; } - loadMessage(messageId); + const thisTab = await browser.tabs.getCurrent(); + const baseTabId = thisTab.openerTabId ?? thisTab.id; + const [header] = await browser.messageDisplay.getDisplayedMessages(baseTabId); + + if (header) { + loadMessage(header.id); + } else { + aiLog("Details popup: no displayed message found"); + } }); async function loadMessage(id) { diff --git a/manifest.json b/manifest.json index 77506b2..2ff6b74 100644 --- a/manifest.json +++ b/manifest.json @@ -24,7 +24,8 @@ "message_display_action": { "default_icon": "resources/img/brain.png", "default_title": "Details", - "default_label": "Details" + "default_label": "Details", + "default_popup": "details.html" }, "background": { "scripts": [ "background.js" ] }, "options_ui": { From 254f0c5ffc1672e071cb38f82c9b8a235416ab70 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 04:31:51 -0500 Subject: [PATCH 045/124] just catching up --- background.js | 15 +++++++++++++++ manifest.json | 3 +-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/background.js b/background.js index 6d0b4e8..d5a5e3a 100644 --- a/background.js +++ b/background.js @@ -369,7 +369,22 @@ async function clearCacheForMessages(idsInput) { icons: { "16": "resources/img/brain.png" } }); + //for the love of god work please + browser.messageDisplayAction.onClicked.addListener(async (tab, info) => { + try { + let header = await browser.messageDisplay.getDisplayedMessages(); + if (!header) { + logger.aiLog("No header, no message loaded?", { debug: true }); + return; + } + const url = browser.runtime.getURL(`details.html?mid=${header.id}`); + await browser.messageDisplayAction.setPopup({ tabId: tab.id, popup: url }); + await browser.messageDisplayAction.openPopup({ tabId: tab.id }); + } catch (err) { + logger.aiLog("Failed to open details popup", { debug: true }); + } + }); browser.menus.onClicked.addListener(async (info, tab) => { if (info.menuItemId === "apply-ai-rules-list" || info.menuItemId === "apply-ai-rules-display") { diff --git a/manifest.json b/manifest.json index 2ff6b74..77506b2 100644 --- a/manifest.json +++ b/manifest.json @@ -24,8 +24,7 @@ "message_display_action": { "default_icon": "resources/img/brain.png", "default_title": "Details", - "default_label": "Details", - "default_popup": "details.html" + "default_label": "Details" }, "background": { "scripts": [ "background.js" ] }, "options_ui": { From aec56aac33a6003348f04ded1cf1b85772dc0d52 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 04:35:46 -0500 Subject: [PATCH 046/124] Revert manifest to version 2 --- manifest.json | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/manifest.json b/manifest.json index 77506b2..0d6db48 100644 --- a/manifest.json +++ b/manifest.json @@ -1,9 +1,9 @@ { - "manifest_version": 3, + "manifest_version": 2, "name": "Sortana", "version": "2.1.0", "default_locale": "en-US", - "browser_specific_settings": { + "applications": { "gecko": { "id": "ai-filter@jordanwages", "strict_min_version": "128.0", @@ -18,7 +18,7 @@ "96": "resources/img/logo96.png", "128": "resources/img/logo128.png" }, - "action": { + "browser_action": { "default_icon": "resources/img/logo32.png" }, "message_display_action": { @@ -40,12 +40,8 @@ "accountsRead", "menus", "scripting", - "tabs" - ], - "host_permissions": [ + "tabs", "*://*/*" ], - "content_security_policy": { - "extension_pages": "script-src 'self'; object-src 'none'; connect-src 'self' http: https:" - } + "content_security_policy": "script-src 'self'; object-src 'none'; connect-src 'self' http: https:" } From 13751b3ab2741b23cbbf5fc0e8e497ed7dea203c Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sun, 6 Jul 2025 18:05:19 -0500 Subject: [PATCH 047/124] Trying new things --- background.js | 24 ++++++++++++++---------- details.js | 14 +++++++------- manifest.json | 6 ++---- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/background.js b/background.js index d5a5e3a..b600169 100644 --- a/background.js +++ b/background.js @@ -258,7 +258,7 @@ async function clearCacheForMessages(idsInput) { logger = await import(browser.runtime.getURL("logger.js")); try { AiClassifier = await import(browser.runtime.getURL("modules/AiClassifier.js")); - logger.aiLog("AiClassifier imported", {debug: true}); + logger.aiLog("AiClassifier imported", { debug: true }); const td = await import(browser.runtime.getURL("resources/js/turndown.js")); TurndownService = td.default || td.TurndownService; } catch (e) { @@ -291,7 +291,7 @@ async function clearCacheForMessages(idsInput) { if (r.stopProcessing) rule.stopProcessing = true; return rule; }) : []; - logger.aiLog("configuration loaded", {debug: true}, store); + logger.aiLog("configuration loaded", { debug: true }, store); storage.onChanged.addListener(async changes => { if (changes.aiRules) { const newRules = changes.aiRules.newValue || []; @@ -304,30 +304,30 @@ async function clearCacheForMessages(idsInput) { if (r.stopProcessing) rule.stopProcessing = true; return rule; }); - logger.aiLog("aiRules updated from storage change", {debug: true}, aiRules); + logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } if (changes.htmlToMarkdown) { htmlToMarkdown = changes.htmlToMarkdown.newValue === true; - logger.aiLog("htmlToMarkdown updated from storage change", {debug: true}, htmlToMarkdown); + logger.aiLog("htmlToMarkdown updated from storage change", { debug: true }, htmlToMarkdown); } if (changes.stripUrlParams) { stripUrlParams = changes.stripUrlParams.newValue === true; - logger.aiLog("stripUrlParams updated from storage change", {debug: true}, stripUrlParams); + logger.aiLog("stripUrlParams updated from storage change", { debug: true }, stripUrlParams); } if (changes.altTextImages) { altTextImages = changes.altTextImages.newValue === true; - logger.aiLog("altTextImages updated from storage change", {debug: true}, altTextImages); + logger.aiLog("altTextImages updated from storage change", { debug: true }, altTextImages); } if (changes.collapseWhitespace) { collapseWhitespace = changes.collapseWhitespace.newValue === true; - logger.aiLog("collapseWhitespace updated from storage change", {debug: true}, collapseWhitespace); + logger.aiLog("collapseWhitespace updated from storage change", { debug: true }, collapseWhitespace); } }); } catch (err) { - logger.aiLog("failed to load config", {level: 'error'}, err); + logger.aiLog("failed to load config", { level: 'error' }, err); } - logger.aiLog("background.js loaded – ready to classify", {debug: true}); + logger.aiLog("background.js loaded – ready to classify", { debug: true }); if (browser.messageDisplayAction) { browser.messageDisplayAction.setTitle({ title: "Details" }); if (browser.messageDisplayAction.setLabel) { @@ -372,7 +372,7 @@ async function clearCacheForMessages(idsInput) { //for the love of god work please browser.messageDisplayAction.onClicked.addListener(async (tab, info) => { try { - let header = await browser.messageDisplay.getDisplayedMessages(); + let header = await browser.messageDisplay.getDisplayedMessages(tab.id); if (!header) { logger.aiLog("No header, no message loaded?", { debug: true }); return; @@ -386,6 +386,10 @@ async function clearCacheForMessages(idsInput) { } }); + browser.messageDisplay.onMessagesDisplayed.addListener(async (tab, displayedMessages) => { + logger.aiLog("Messages displayed!", { debug: true }, displayedMessages); + }); + browser.menus.onClicked.addListener(async (info, tab) => { if (info.menuItemId === "apply-ai-rules-list" || info.menuItemId === "apply-ai-rules-display") { const ids = info.selectedMessages?.messages?.map(m => m.id) || diff --git a/details.js b/details.js index 016783a..6269dae 100644 --- a/details.js +++ b/details.js @@ -8,14 +8,14 @@ document.addEventListener("DOMContentLoaded", async () => { } const thisTab = await browser.tabs.getCurrent(); - const baseTabId = thisTab.openerTabId ?? thisTab.id; - const [header] = await browser.messageDisplay.getDisplayedMessages(baseTabId); + //const baseTabId = thisTab.openerTabId ?? thisTab.id; + //const [header] = await browser.messageDisplay.getDisplayedMessages(baseTabId); - if (header) { - loadMessage(header.id); - } else { - aiLog("Details popup: no displayed message found"); - } + //if (header) { + // loadMessage(header.id); + //} else { + // aiLog("Details popup: no displayed message found"); + //} }); async function loadMessage(id) { diff --git a/manifest.json b/manifest.json index 0d6db48..fbd37ae 100644 --- a/manifest.json +++ b/manifest.json @@ -40,8 +40,6 @@ "accountsRead", "menus", "scripting", - "tabs", - "*://*/*" - ], - "content_security_policy": "script-src 'self'; object-src 'none'; connect-src 'self' http: https:" + "tabs" + ] } From caf18ed5ab1c5db206303b2a79f0578538c998ce Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 7 Jul 2025 00:14:47 -0500 Subject: [PATCH 048/124] Add message for displayed messages and convert details to module --- background.js | 9 +++++++++ details.html | 2 +- details.js | 15 ++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/background.js b/background.js index b600169..d58adf7 100644 --- a/background.js +++ b/background.js @@ -501,6 +501,15 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("failed to collect details", { level: 'error' }, e); return { subject: '', results: [] }; } + } else if (msg?.type === "sortana:getDisplayedMessages") { + try { + const [tab] = await browser.tabs.query({ active: true, currentWindow: true }); + const messages = await browser.messageDisplay.getDisplayedMessages(tab?.id); + return { messages }; + } catch (e) { + logger.aiLog("failed to get displayed messages", { level: 'error' }, e); + return { messages: [] }; + } } else if (msg?.type === "sortana:clearCacheForMessage") { try { await clearCacheForMessages([msg.id]); diff --git a/details.html b/details.html index 1502471..d15a3c9 100644 --- a/details.html +++ b/details.html @@ -15,6 +15,6 @@
- + diff --git a/details.js b/details.js index 6269dae..ca53cbe 100644 --- a/details.js +++ b/details.js @@ -1,12 +1,9 @@ -document.addEventListener("DOMContentLoaded", async () => { - const aiLog = (await import(browser.runtime.getURL("logger.js"))).aiLog; - - const qMid = parseInt(new URLSearchParams(location.search).get("mid"), 10); - if (!isNaN(qMid)) { - loadMessage(qMid); - return; - } +const aiLog = (await import(browser.runtime.getURL("logger.js"))).aiLog; +const qMid = parseInt(new URLSearchParams(location.search).get("mid"), 10); +if (!isNaN(qMid)) { + loadMessage(qMid); +} else { const thisTab = await browser.tabs.getCurrent(); //const baseTabId = thisTab.openerTabId ?? thisTab.id; //const [header] = await browser.messageDisplay.getDisplayedMessages(baseTabId); @@ -16,7 +13,7 @@ document.addEventListener("DOMContentLoaded", async () => { //} else { // aiLog("Details popup: no displayed message found"); //} -}); +} async function loadMessage(id) { const storage = (globalThis.messenger ?? browser).storage; From 51816d8a19d7c8976ada797b5116c711700b97b2 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 7 Jul 2025 00:23:54 -0500 Subject: [PATCH 049/124] Use getDisplayedMessages to load message --- details.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/details.js b/details.js index ca53cbe..1e9023f 100644 --- a/details.js +++ b/details.js @@ -4,15 +4,14 @@ const qMid = parseInt(new URLSearchParams(location.search).get("mid"), 10); if (!isNaN(qMid)) { loadMessage(qMid); } else { - const thisTab = await browser.tabs.getCurrent(); - //const baseTabId = thisTab.openerTabId ?? thisTab.id; - //const [header] = await browser.messageDisplay.getDisplayedMessages(baseTabId); - - //if (header) { - // loadMessage(header.id); - //} else { - // aiLog("Details popup: no displayed message found"); - //} + const { messages } = await browser.runtime.sendMessage({ + type: "sortana:getDisplayedMessages", + }); + if (messages && messages[0]) { + loadMessage(messages[0].id); + } else { + aiLog("Details popup: no displayed message found"); + } } async function loadMessage(id) { From 6a85dbb2eb12ff072871c4aa429e0bdf89b1f310 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 7 Jul 2025 21:46:21 -0500 Subject: [PATCH 050/124] Going back to what works. Manifest v3 and its consequences have been a disaster for the human race. --- background.js | 283 +++++++++++++++++++++++--------------------------- details.js | 11 +- manifest.json | 3 +- 3 files changed, 144 insertions(+), 153 deletions(-) diff --git a/background.js b/background.js index d58adf7..59bac43 100644 --- a/background.js +++ b/background.js @@ -126,7 +126,7 @@ function buildEmailText(full) { const attachments = []; collectText(full, bodyParts, attachments); const headers = Object.entries(full.headers || {}) - .map(([k,v]) => `${k}: ${v.join(' ')}`) + .map(([k, v]) => `${k}: ${v.join(' ')}`) .join('\n'); const attachInfo = `Attachments: ${attachments.length}` + (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : ""); const combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); @@ -369,35 +369,14 @@ async function clearCacheForMessages(idsInput) { icons: { "16": "resources/img/brain.png" } }); - //for the love of god work please - browser.messageDisplayAction.onClicked.addListener(async (tab, info) => { - try { - let header = await browser.messageDisplay.getDisplayedMessages(tab.id); - if (!header) { - logger.aiLog("No header, no message loaded?", { debug: true }); - return; - } - - const url = browser.runtime.getURL(`details.html?mid=${header.id}`); - await browser.messageDisplayAction.setPopup({ tabId: tab.id, popup: url }); - await browser.messageDisplayAction.openPopup({ tabId: tab.id }); - } catch (err) { - logger.aiLog("Failed to open details popup", { debug: true }); - } - }); - - browser.messageDisplay.onMessagesDisplayed.addListener(async (tab, displayedMessages) => { - logger.aiLog("Messages displayed!", { debug: true }, displayedMessages); - }); - browser.menus.onClicked.addListener(async (info, tab) => { if (info.menuItemId === "apply-ai-rules-list" || info.menuItemId === "apply-ai-rules-display") { const ids = info.selectedMessages?.messages?.map(m => m.id) || - (info.messageId ? [info.messageId] : []); + (info.messageId ? [info.messageId] : []); await applyAiRules(ids); } else if (info.menuItemId === "clear-ai-cache-list" || info.menuItemId === "clear-ai-cache-display") { const ids = info.selectedMessages?.messages?.map(m => m.id) || - (info.messageId ? [info.messageId] : []); + (info.messageId ? [info.messageId] : []); await clearCacheForMessages(ids); } else if (info.menuItemId === "view-ai-reason-list" || info.menuItemId === "view-ai-reason-display") { const [header] = await browser.messageDisplay.getDisplayedMessages(tab.id); @@ -417,141 +396,143 @@ async function clearCacheForMessages(idsInput) { } if (msg?.type === "sortana:test") { - const { text = "", criterion = "" } = msg; - logger.aiLog("sortana:test – text", {debug: true}, text); - logger.aiLog("sortana:test – criterion", {debug: true}, criterion); + const { text = "", criterion = "" } = msg; + logger.aiLog("sortana:test – text", { debug: true }, text); + logger.aiLog("sortana:test – criterion", { debug: true }, criterion); - try { - logger.aiLog("Calling AiClassifier.classifyText()", {debug: true}); - const result = await AiClassifier.classifyText(text, criterion); - logger.aiLog("classify() returned", {debug: true}, result); - return { match: result }; - } - catch (err) { - logger.aiLog("Error in classify()", {level: 'error'}, err); - // rethrow so the caller sees the failure - throw err; - } - } else if (msg?.type === "sortana:clearCacheForDisplayed") { - try { - const msgs = await browser.messageDisplay.getDisplayedMessages(); - const ids = msgs.map(m => m.id); - await clearCacheForMessages(ids); - } catch (e) { - logger.aiLog("failed to clear cache from message script", { level: 'error' }, e); - } - } else if (msg?.type === "sortana:getReasons") { - try { - const id = msg.id; - const hdr = await messenger.messages.get(id); - const subject = hdr?.subject || ""; - if (!aiRules.length) { - const { aiRules: stored } = await storage.local.get("aiRules"); - aiRules = Array.isArray(stored) ? stored.map(r => { - if (r.actions) return r; - const actions = []; - if (r.tag) actions.push({ type: 'tag', tagKey: r.tag }); - if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); - const rule = { criterion: r.criterion, actions }; - if (r.stopProcessing) rule.stopProcessing = true; - return rule; - }) : []; + try { + logger.aiLog("Calling AiClassifier.classifyText()", { debug: true }); + const result = await AiClassifier.classifyText(text, criterion); + logger.aiLog("classify() returned", { debug: true }, result); + return { match: result }; } - const reasons = []; - for (const rule of aiRules) { - const key = await AiClassifier.buildCacheKey(id, rule.criterion); - const reason = AiClassifier.getReason(key); - if (reason) { - reasons.push({ criterion: rule.criterion, reason }); + catch (err) { + logger.aiLog("Error in classify()", { level: 'error' }, err); + // rethrow so the caller sees the failure + throw err; + } + } else if (msg?.type === "sortana:clearCacheForDisplayed") { + try { + const msgs = await browser.messageDisplay.getDisplayedMessages(); + const ids = msgs.map(m => m.id); + await clearCacheForMessages(ids); + } catch (e) { + logger.aiLog("failed to clear cache from message script", { level: 'error' }, e); + } + } else if (msg?.type === "sortana:getReasons") { + try { + const id = msg.id; + const hdr = await messenger.messages.get(id); + const subject = hdr?.subject || ""; + if (!aiRules.length) { + const { aiRules: stored } = await storage.local.get("aiRules"); + aiRules = Array.isArray(stored) ? stored.map(r => { + if (r.actions) return r; + const actions = []; + if (r.tag) actions.push({ type: 'tag', tagKey: r.tag }); + if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); + const rule = { criterion: r.criterion, actions }; + if (r.stopProcessing) rule.stopProcessing = true; + return rule; + }) : []; } - } - return { subject, reasons }; - } catch (e) { - logger.aiLog("failed to collect reasons", { level: 'error' }, e); - return { subject: '', reasons: [] }; - } - } else if (msg?.type === "sortana:getDetails") { - try { - const id = msg.id; - const hdr = await messenger.messages.get(id); - const subject = hdr?.subject || ""; - if (!aiRules.length) { - const { aiRules: stored } = await storage.local.get("aiRules"); - aiRules = Array.isArray(stored) ? stored.map(r => { - if (r.actions) return r; - const actions = []; - if (r.tag) actions.push({ type: 'tag', tagKey: r.tag }); - if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); - const rule = { criterion: r.criterion, actions }; - if (r.stopProcessing) rule.stopProcessing = true; - return rule; - }) : []; - } - const results = []; - for (const rule of aiRules) { - const key = await AiClassifier.buildCacheKey(id, rule.criterion); - const matched = AiClassifier.getCachedResult(key); - const reason = AiClassifier.getReason(key); - if (matched !== null || reason) { - results.push({ criterion: rule.criterion, matched, reason }); + const reasons = []; + for (const rule of aiRules) { + const key = await AiClassifier.buildCacheKey(id, rule.criterion); + const reason = AiClassifier.getReason(key); + if (reason) { + reasons.push({ criterion: rule.criterion, reason }); + } } + return { subject, reasons }; + } catch (e) { + logger.aiLog("failed to collect reasons", { level: 'error' }, e); + return { subject: '', reasons: [] }; } - return { subject, results }; - } catch (e) { - logger.aiLog("failed to collect details", { level: 'error' }, e); - return { subject: '', results: [] }; - } - } else if (msg?.type === "sortana:getDisplayedMessages") { - try { - const [tab] = await browser.tabs.query({ active: true, currentWindow: true }); - const messages = await browser.messageDisplay.getDisplayedMessages(tab?.id); - return { messages }; - } catch (e) { - logger.aiLog("failed to get displayed messages", { level: 'error' }, e); - return { messages: [] }; - } - } else if (msg?.type === "sortana:clearCacheForMessage") { - try { - await clearCacheForMessages([msg.id]); - return { ok: true }; - } catch (e) { - logger.aiLog("failed to clear cache for message", { level: 'error' }, e); - return { ok: false }; - } - } else if (msg?.type === "sortana:getQueueCount") { - return { count: queuedCount + (processing ? 1 : 0) }; - } else if (msg?.type === "sortana:getTiming") { - const t = timingStats; - const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0; - return { - count: queuedCount + (processing ? 1 : 0), - current: currentStart ? Date.now() - currentStart : -1, - last: t.last, - runs: t.count, - average: t.mean, - total: t.total, - stddev: std - }; - } else { - logger.aiLog("Unknown message type, ignoring", {level: 'warn'}, msg?.type); - } -}); + } else if (msg?.type === "sortana:getDetails") { + try { + const id = msg.id; + const hdr = await messenger.messages.get(id); + const subject = hdr?.subject || ""; + if (!aiRules.length) { + const { aiRules: stored } = await storage.local.get("aiRules"); + aiRules = Array.isArray(stored) ? stored.map(r => { + if (r.actions) return r; + const actions = []; + if (r.tag) actions.push({ type: 'tag', tagKey: r.tag }); + if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); + const rule = { criterion: r.criterion, actions }; + if (r.stopProcessing) rule.stopProcessing = true; + return rule; + }) : []; + } + const results = []; + for (const rule of aiRules) { + const key = await AiClassifier.buildCacheKey(id, rule.criterion); + const matched = AiClassifier.getCachedResult(key); + const reason = AiClassifier.getReason(key); + if (matched !== null || reason) { + results.push({ criterion: rule.criterion, matched, reason }); + } + } + return { subject, results }; + } catch (e) { + logger.aiLog("failed to collect details", { level: 'error' }, e); + return { subject: '', results: [] }; + } + } else if (msg?.type === "sortana:getDisplayedMessages") { + try { + const [tab] = await browser.tabs.query({ active: true, currentWindow: true }); + const messages = await browser.messageDisplay.getDisplayedMessages(tab?.id); + const ids = messages.map(hdr => hdr.id); -// Automatically classify new messages -if (typeof messenger !== "undefined" && messenger.messages?.onNewMailReceived) { - messenger.messages.onNewMailReceived.addListener(async (folder, messages) => { - logger.aiLog("onNewMailReceived", {debug: true}, messages); - const ids = (messages?.messages || messages || []).map(m => m.id ?? m); - await applyAiRules(ids); + return { ids }; + } catch (e) { + logger.aiLog("failed to get displayed messages", { level: 'error' }, e); + return { messages: [] }; + } + } else if (msg?.type === "sortana:clearCacheForMessage") { + try { + await clearCacheForMessages([msg.id]); + return { ok: true }; + } catch (e) { + logger.aiLog("failed to clear cache for message", { level: 'error' }, e); + return { ok: false }; + } + } else if (msg?.type === "sortana:getQueueCount") { + return { count: queuedCount + (processing ? 1 : 0) }; + } else if (msg?.type === "sortana:getTiming") { + const t = timingStats; + const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0; + return { + count: queuedCount + (processing ? 1 : 0), + current: currentStart ? Date.now() - currentStart : -1, + last: t.last, + runs: t.count, + average: t.mean, + total: t.total, + stddev: std + }; + } else { + logger.aiLog("Unknown message type, ignoring", { level: 'warn' }, msg?.type); + } }); -} else { - logger.aiLog("messenger.messages API unavailable, skipping new mail listener", { level: 'warn' }); -} -// Catch any unhandled rejections -window.addEventListener("unhandledrejection", ev => { - logger.aiLog("Unhandled promise rejection", {level: 'error'}, ev.reason); -}); + // Automatically classify new messages + if (typeof messenger !== "undefined" && messenger.messages?.onNewMailReceived) { + messenger.messages.onNewMailReceived.addListener(async (folder, messages) => { + logger.aiLog("onNewMailReceived", { debug: true }, messages); + const ids = (messages?.messages || messages || []).map(m => m.id ?? m); + await applyAiRules(ids); + }); + } else { + logger.aiLog("messenger.messages API unavailable, skipping new mail listener", { level: 'warn' }); + } + + // Catch any unhandled rejections + window.addEventListener("unhandledrejection", ev => { + logger.aiLog("Unhandled promise rejection", { level: 'error' }, ev.reason); + }); browser.runtime.onInstalled.addListener(async ({ reason }) => { if (reason === "install") { diff --git a/details.js b/details.js index 1e9023f..f84ebf4 100644 --- a/details.js +++ b/details.js @@ -10,7 +10,16 @@ if (!isNaN(qMid)) { if (messages && messages[0]) { loadMessage(messages[0].id); } else { - aiLog("Details popup: no displayed message found"); + const tabs = await browser.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs[0]?.id; + const msgs = tabId ? await browser.messageDisplay.getDisplayedMessages(tabId) : []; + let id = msgs[0]?.id; + if (id) { + loadMessage(id); + } + else { + aiLog("Details popup: no displayed message found"); + } } } diff --git a/manifest.json b/manifest.json index fbd37ae..36b94b0 100644 --- a/manifest.json +++ b/manifest.json @@ -24,7 +24,8 @@ "message_display_action": { "default_icon": "resources/img/brain.png", "default_title": "Details", - "default_label": "Details" + "default_label": "Details", + "default_popup": "details.html" }, "background": { "scripts": [ "background.js" ] }, "options_ui": { From c7333482ce0fe646e7eb601fa0f9700f648ac8ee Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 7 Jul 2025 22:07:00 -0500 Subject: [PATCH 051/124] Allow selective data export and import --- options/dataTransfer.js | 45 +++++++++++++++++++++++++++++++++++++++++ options/options.html | 17 ++++++++++++++++ options/options.js | 19 +++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 options/dataTransfer.js diff --git a/options/dataTransfer.js b/options/dataTransfer.js new file mode 100644 index 0000000..b289c02 --- /dev/null +++ b/options/dataTransfer.js @@ -0,0 +1,45 @@ +"use strict"; +const storage = (globalThis.messenger ?? browser).storage; +const KEY_GROUPS = { + settings: [ + 'endpoint', + 'templateName', + 'customTemplate', + 'customSystemPrompt', + 'aiParams', + 'debugLogging', + 'htmlToMarkdown', + 'stripUrlParams', + 'altTextImages', + 'collapseWhitespace' + ], + rules: ['aiRules'], + cache: ['aiCache'] +}; + +function collectKeys(categories = Object.keys(KEY_GROUPS)) { + return categories.flatMap(cat => KEY_GROUPS[cat] || []); +} + +export async function exportData(categories) { + const data = await storage.local.get(collectKeys(categories)); + const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'sortana-export.json'; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); +} + +export async function importData(file, categories) { + const text = await file.text(); + const parsed = JSON.parse(text); + const data = {}; + for (const key of collectKeys(categories)) { + if (key in parsed) data[key] = parsed[key]; + } + await storage.local.set(data); +} diff --git a/options/options.html b/options/options.html index f40cda2..186e2b6 100644 --- a/options/options.html +++ b/options/options.html @@ -207,6 +207,23 @@ +
+ +
+ + + +
+
+
+

+ +

+

+ + +

+
diff --git a/options/options.js b/options/options.js index 2350efb..ca25eb4 100644 --- a/options/options.js +++ b/options/options.js @@ -2,6 +2,7 @@ document.addEventListener('DOMContentLoaded', async () => { const storage = (globalThis.messenger ?? browser).storage; const logger = await import(browser.runtime.getURL('logger.js')); const AiClassifier = await import(browser.runtime.getURL('modules/AiClassifier.js')); + const dataTransfer = await import(browser.runtime.getURL('options/dataTransfer.js')); const defaults = await storage.local.get([ 'endpoint', 'templateName', @@ -395,6 +396,24 @@ document.addEventListener('DOMContentLoaded', async () => { await AiClassifier.clearCache(); cacheCountEl.textContent = '0'; }); + + function selectedCategories() { + return [...document.querySelectorAll('.transfer-category:checked')].map(el => el.value); + } + + document.getElementById('export-data').addEventListener('click', () => { + dataTransfer.exportData(selectedCategories()); + }); + + const importInput = document.getElementById('import-file'); + document.getElementById('import-data').addEventListener('click', () => importInput.click()); + importInput.addEventListener('change', async () => { + if (importInput.files.length) { + await dataTransfer.importData(importInput.files[0], selectedCategories()); + location.reload(); + } + }); + initialized = true; document.getElementById('save').addEventListener('click', async () => { From 97628c693ba80c96d855cf2d79ce222b7e8d5448 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 7 Jul 2025 23:54:52 -0500 Subject: [PATCH 052/124] Changing up imagery --- ai-filter.sln | 7 +++---- resources/img/average-16.png | Bin 0 -> 416 bytes resources/img/average-32.png | Bin 0 -> 794 bytes resources/img/average-64.png | Bin 0 -> 1506 bytes resources/img/brain.png | Bin 13993 -> 0 bytes resources/img/busy.png | Bin 3603 -> 0 bytes resources/img/circle-16.png | Bin 0 -> 389 bytes resources/img/circle-32.png | Bin 0 -> 724 bytes resources/img/circle-64.png | Bin 0 -> 1566 bytes resources/img/clipboarddata-16.png | Bin 0 -> 314 bytes resources/img/clipboarddata-32.png | Bin 0 -> 543 bytes resources/img/clipboarddata-64.png | Bin 0 -> 989 bytes resources/img/done.png | Bin 3543 -> 0 bytes resources/img/download-16.png | Bin 0 -> 345 bytes resources/img/download-32.png | Bin 0 -> 571 bytes resources/img/download-64.png | Bin 0 -> 1006 bytes resources/img/error.png | Bin 2921 -> 0 bytes resources/img/eye-16.png | Bin 0 -> 371 bytes resources/img/eye-32.png | Bin 0 -> 733 bytes resources/img/eye-64.png | Bin 0 -> 1394 bytes resources/img/flag-16.png | Bin 0 -> 300 bytes resources/img/flag-32.png | Bin 0 -> 475 bytes resources/img/flag-64.png | Bin 0 -> 800 bytes resources/img/gear-16.png | Bin 0 -> 462 bytes resources/img/gear-32.png | Bin 0 -> 993 bytes resources/img/gear-64.png | Bin 0 -> 2169 bytes resources/img/reply-16.png | Bin 0 -> 289 bytes resources/img/reply-32.png | Bin 0 -> 432 bytes resources/img/reply-64.png | Bin 0 -> 750 bytes resources/img/settings-16.png | Bin 0 -> 421 bytes resources/img/settings-32.png | Bin 0 -> 787 bytes resources/img/settings-64.png | Bin 0 -> 1489 bytes resources/img/trash-16.png | Bin 0 -> 390 bytes resources/img/trash-32.png | Bin 0 -> 631 bytes resources/img/trash-64.png | Bin 0 -> 1126 bytes resources/img/upload-16.png | Bin 0 -> 352 bytes resources/img/upload-32.png | Bin 0 -> 565 bytes resources/img/upload-64.png | Bin 0 -> 1006 bytes resources/svg/average.svg | 3 +++ resources/svg/circle.svg | 3 +++ resources/svg/clipboarddata.svg | 3 +++ resources/svg/download.svg | 4 ++++ resources/svg/eye.svg | 4 ++++ resources/svg/flag.svg | 3 +++ resources/svg/gear.svg | 11 +++++++++++ resources/svg/reply.svg | 4 ++++ resources/svg/settings.svg | 3 +++ resources/svg/trash.svg | 3 +++ resources/svg/upload.svg | 4 ++++ resources/svg2img.ps1 | 28 ++++++++++++++++++++++++++++ 50 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 resources/img/average-16.png create mode 100644 resources/img/average-32.png create mode 100644 resources/img/average-64.png delete mode 100644 resources/img/brain.png delete mode 100644 resources/img/busy.png create mode 100644 resources/img/circle-16.png create mode 100644 resources/img/circle-32.png create mode 100644 resources/img/circle-64.png create mode 100644 resources/img/clipboarddata-16.png create mode 100644 resources/img/clipboarddata-32.png create mode 100644 resources/img/clipboarddata-64.png delete mode 100644 resources/img/done.png create mode 100644 resources/img/download-16.png create mode 100644 resources/img/download-32.png create mode 100644 resources/img/download-64.png delete mode 100644 resources/img/error.png create mode 100644 resources/img/eye-16.png create mode 100644 resources/img/eye-32.png create mode 100644 resources/img/eye-64.png create mode 100644 resources/img/flag-16.png create mode 100644 resources/img/flag-32.png create mode 100644 resources/img/flag-64.png create mode 100644 resources/img/gear-16.png create mode 100644 resources/img/gear-32.png create mode 100644 resources/img/gear-64.png create mode 100644 resources/img/reply-16.png create mode 100644 resources/img/reply-32.png create mode 100644 resources/img/reply-64.png create mode 100644 resources/img/settings-16.png create mode 100644 resources/img/settings-32.png create mode 100644 resources/img/settings-64.png create mode 100644 resources/img/trash-16.png create mode 100644 resources/img/trash-32.png create mode 100644 resources/img/trash-64.png create mode 100644 resources/img/upload-16.png create mode 100644 resources/img/upload-32.png create mode 100644 resources/img/upload-64.png create mode 100644 resources/svg/average.svg create mode 100644 resources/svg/circle.svg create mode 100644 resources/svg/clipboarddata.svg create mode 100644 resources/svg/download.svg create mode 100644 resources/svg/eye.svg create mode 100644 resources/svg/flag.svg create mode 100644 resources/svg/gear.svg create mode 100644 resources/svg/reply.svg create mode 100644 resources/svg/settings.svg create mode 100644 resources/svg/trash.svg create mode 100644 resources/svg/upload.svg create mode 100644 resources/svg2img.ps1 diff --git a/ai-filter.sln b/ai-filter.sln index 7922392..7818aed 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -47,13 +47,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_ EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{68A87938-5C2B-49F5-8AAA-8A34FBBFD854}" + ProjectSection(SolutionItems) = preProject + resources\svg2img.ps1 = resources\svg2img.ps1 + EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-1755-4A95-A11B-6C90C701C5BF}" ProjectSection(SolutionItems) = preProject - resources\img\brain.png = resources\img\brain.png - resources\img\busy.png = resources\img\busy.png - resources\img\done.png = resources\img\done.png - resources\img\error.png = resources\img\error.png resources\img\full-logo.png = resources\img\full-logo.png resources\img\logo.png = resources\img\logo.png resources\img\logo128.png = resources\img\logo128.png diff --git a/resources/img/average-16.png b/resources/img/average-16.png new file mode 100644 index 0000000000000000000000000000000000000000..ff202ff88b987856d28c37c894f1a362af2b0b00 GIT binary patch literal 416 zcmV;R0bl-!P)H zK~y-6ozuH6MPU#H;BOjr z&X4U9JDKFqT3Pe2HM9RJBK+s8s%i>%1K*G>tY9*s8=OYOL-K7LU{*xbg+c7&1y?xA z>L#|2PM+Hlp%f3{9JiP+8z;YjdjvW?yo|7eQOrfeQ$wNjzLb+H){`qIb*E*iG6n$up7h@*nKOikV3^&9aaHX`D%VJ1<|V5^MK zWcyRBZZ`fh#DzbImvV?>pZ|kB?|c49Jd8W+)>uJyGS@v88^nt#KEvRL28HcTI(fl0 zj&YLbXY7{Zh)`iVSGd+&RX(O=?~kd<2B#76*su-#m~P7amwE%VNkkY)Rgm5Q0000< KMNUMnLSTY>O0#DG literal 0 HcmV?d00001 diff --git a/resources/img/average-32.png b/resources/img/average-32.png new file mode 100644 index 0000000000000000000000000000000000000000..e0fe4b46d0d2d0de6ca4ff004a33dd5acb80462d GIT binary patch literal 794 zcmV+#1LgdQP)7;9myn zimd}B5g`w87jOYs4v=Pz??46EqcKK-pZe?%@EBMEsyf#X z><3N(2MqQ-a2vR*+ui_~r4E!g;4^{@81g^BrM$Uf zVoZAi#)7zjW4CTyN`9;ja+`4JSOB{62)JgP+F7}u?RQMP%TWO&rS?_9FFQb9*7;XD z_XV#+{Qy%&?WPHrhg_eHn79)$0S^`QVTp_4!Opz?yy|4o8H`#TV+>hDh zvI6X?`^d01$^Dp3uBHGHsgoSD0D~FiW#E~TH-Mu#$!*M4vAu(rD6=Hnv$&mv%Sgf{ z^}QD>!%141I=t0Vi;y2SamQlBA1St55Q9IoM^623fa_9!; zjnj6>Pmmnx0=y&7ajcGGo`C5#$gdjm8qgoa1sp@ogpeGmBRLc9jq$4ihqCbNno_TC zZvf|k>p+Q?eIjX8{ibvfI8D-=rZx|p5s@lcW|Rl(oK{<23+zhP_=aiAqdTS~B{s=+ zk`kQ&W`NZI=?ZWk7{@EMJOU~)@{D)g1MJh$v(+SBmjn8vcoBFmA`6-C>mOM3&uAy? YFQTTm<6Y?dp#T5?07*qoM6N<$f-$sWFaQ7m literal 0 HcmV?d00001 diff --git a/resources/img/average-64.png b/resources/img/average-64.png new file mode 100644 index 0000000000000000000000000000000000000000..bec89df178cb32f906ae102e43d4ff8284711374 GIT binary patch literal 1506 zcmV<81s(c{P)d2v}$TK3Ik$)eORZ2I4U|Y@fQOcb7!y=VZ7c=t_^^B1y?f8h?(ELpyD2A`hdcM|Ip_O5Gqdw+ zmDZXESsrS10F9Ohh5)5hn^H=}>(@O2N~sQ|)DOS{Fr$=uvFiWzJO(JGI)HItccgkw zYu&l3QzQXOsU~1M&%o@<9zX+|wAOb4n({hI2%-fz2aLgcN)3K~O369&j9Z zG==&sa0R#u+#(qK0NQ}3fStf|1cPLfEC2&q>#q~~{AJ)Zt@WRxl^j{$7y`FtY6Un4 z9058VYUI&BqBGn|gg?TR?r#CMF7F7E^c%g4-*_8nUCrBB2SjAtH`7v zh6RH|Wy*Z)xl@Av$xH+Qo2VW6SL&U8*USs8Aij2(2U@Za@U{@Zxw3+f*q9WaJ!I0i z?F#P`>P}nqb4jixZM8@-?l)>%n7m9alU-%ae8g6_VB?Zg7a-UyjgP1^Wd|Rzc_U_L z8Q9>`1-hx-z5T`5j53N+s-0l@Y-;ugQa1azT#5bONU+ILBPIg23w75s1Y(r1&rbj^ z2!B`l{2v+3Ulr;kOhy8v{k>ZmgQ<}^X1*d4apIn(iGWt2Zc(UnmJs|H=FA_+k@<@J zEz~_CDHeJ-S`c_AF%ckTyDg8Sg27AKUXT<`XM>XRlUbwjILS#XQmj(9Ft&hMp>9`} zND3J6BRSx8;J6`D0`Ss?$Zeq6nSjvXkN{-Z zpqptM%QFF|1m7pD&mG@0g8voZrQ8IB4!#l`EMY#SEfW0m$VX$RQ$3&3$rjRIc`Fip7x0@9!#uFr z8v&uS!zBcN-4Mq+sqHvAwOxvuF1XoCnb*KECtD6qlg{UtfTIO~|H#niHSitKS;BRI6U2K-L-RoSNrc5^D(n;C!Rn7ogy^Jjxn!W9O{$*gaj=)DrF}=-9KpyZh zJ9~hW1T(U{pl^q^tgAX-1emVDW<8~C2Hpfd0WMLy(JGZ?;3Duba0tt0hL5s32r8wT z3GV*IzpeAOho_XbMPR1HyD0i+N!noavEHAS=V2Vrjd33?QoL(53T<1VIFT)r~m)}07*qo IM6N<$g6_7zO#lD@ literal 0 HcmV?d00001 diff --git a/resources/img/brain.png b/resources/img/brain.png deleted file mode 100644 index 296bb646386cd6cbcbc028e261e84bc130a313a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13993 zcmeIZhgVbS+bH_(B!nOdM2b=b!k{<~iYSC$0$6YqMNv@{3F-(2g*b>bflV>k5FBUJ z!60!|1O-76r4vNK85K|?7Fw_XA|*<%xzA>P-(BB5-yd+-S?6Re7kj_u>Fs^q?FojALU#Z(-9a{ zJiNwzrQ#Q=E>%xMV-iyYkZ8>c=gr~ods=pEn&Ku1=iDOzrCfX#e;3f1m`sOCM0sy20_$!TXP6>#Ls@JowFr zx3(N(fi14AzHlY^COfkcF%jY-^Q_rB{ks_mLr7po)`<(vY@L~4nv1g8LsT53N8c?l zFZ_2HcYemtY?SCC5<+Ijwd*8l-ee8{b*7WkB0U2U4T-|`WUqM`QN_<}mFU1S8gDUs zjrCHkI?1J(rOwIMjg|Zr$Z2Vw{;-^Ntyscp7jikCA>DK!BGKYvnvrleJ%G199b+Lp zmFl@C(L5e=Z^<@C6vTYV<(=fOG81LnunF2+($=}pu2ZC1uy51>*&Mg4*@h3t#CT?h zM4Ldq--7LGtvkPM^?$%+?ejEGEO7pieyv0D=N4ke-h%Ae4~tn})^7?6pCB-_9N;DL zO)thQ&9*6v1sG(R3VJ+Hvpkp*r&0CC!{40oOtVU-x+~c0)i}@t<~e$LG_Z0-UY$H zb;A>oit$6<`|`{hNm?|3!fRHlCz{XRTHR*@#?@m2fD+DRZc|U}aL5X3i6B7v0w}7U zh0CM#6NBENiZ_>r?yJNoD8Q|OW;2)fs^V&mHs%Ik#EGYz($%0PgHuFf7Iqwt2YM9Z)+(-brpU}o{RwQje2bk;Hw&pjH9t6_7% zMfQq1fvx949^DWTtf@O$si`}j5hxn;~x`eEay4|L-zzt%M?RN&UP@Zeas8GU(-fZ3xo8g&@HJn|m zm+<@#Mv9?IwNv++WbhqI!--%dvMIW|QRa@+_FwBws=wBVx}N@W#`${uDR4XmoX#VjMT9>2BzEUUA@$7a_ z#zv_~Ab+kyYNK#*8J(6pv+naipK#(bW-3k^__(z1QtYdZx02VW@m_3^$J&LCl)VG?0qV;Q1 zsViuBB&B&7sKQKo%tGG<`Mc@uW<9!028dF%mwbRIwxhy}tJde-EOt-0&k5j9UX5P$ry5S89CIVk|CAG>nYfXj%zM*gEpDI)F9Qsy4tZQC* zVP17(Tk#cM?@jmSY!o;~#cL%Afj*Lex9>=@OrIko6Z??aG3==<&oKYZa72T*o!#c! zdxrkLHnPrvtOv^#B?-KANsm>mqtiD`tc$JlYgj(1%j+$VHn^8r$Nb}~cl2K=*tVxu z$UaTq{cEP6ZOVo&N7pKhKu8JasveK8l-w0CkTW3)@!s&lGu46K=cqV~VAx~%z56@r z`vM~mAGBMG`pkkEu*m4cod@@M^N?j!Q&}(osdj4I1|nw>KG$wwi0U-(3Kb5Lo;Ay2 zQ4oGVSQw@Putsa&D`TkPE7*=bjp_kqj;NE&MBz00Wg6DA2rX%|3D#qQTy@vyFLXg@ zkO+hqDxB2~5$aL|t(iuVjbS`stTCb=VR(BR_e8lG$vO~4u7KB5Eu45J=Q&B_1#zXQ zf$@4QLS{VVB0 zXOzFHTU0`GlndhSFx_(DZhOYJp(yv3tz*yX=AEm%6IpZVlMo{z)jt#!LKjo`tq}c2 z{@G4G(&2^dUG&v{;EA4YAN<1$KiIc1Y>~Kf=V>ZLg^i)n`yC9vMWip=RCn9E)`s5- ztSZ)gj2k~g$An8MsJX z@XoJqVuJjYG2EOlsn{0ihev|t>N8X+-7!gN@*swAk#I|=-{;HQLJV;p&EbH^EdDBCO3*~GZ-?-)r2Mezd;cgE8rnH*8ZK1# zf8^L?g5iZ>bKeI>sm$xxt3;RcS4@qa%(mm_$q$a`9*C_Fn)K_LovS3XJ}tXD>zc~R zo`Q@&h9yq>shhu+mJIFc^B^pgG*s*4dgSv|kIbJE7Pghit;fJzLM3{`-$Z8OzBs#` zP<$X+YC@zD_xRQ%DP0Ncv0|a)r42qkSp^ZD@1z-H5#sdQ{`cCzFXq~HD^J|8tQwmQ zMFs4-dV4gcUu}3pGRXynEr1A;Xc4JJ$j5rbXY~$+TI)6o> z?$p(CDny|WEnlaOBdJjBYiRmhvJ0pO7ZgG-+wUhYXbd}zXFv-%Hk*5X|`$xkpa~DO@z$?SW?8jQu*dVi2 zEh?xI!|G3vU-?(@pmnxY;XdEB zz3&q1MV7)QNl0+YKwm_ZO8F=iSC)8WM~Ex>BJ+I4J!4{JWq6%DhWew(qZ^HFr8 zIgMZ0zWz}R*z?Yww7nA_j^@9T2Gv*DI0qB4?2|r)F4t&kD<6K+N-k0-sA}YvOcwa^Fz*4Y~BF zbV`A+*rS%AHL?GT3F1(R%-8?-F?_cX=nJkTwI({FdH(DwgJk`psdj>x-Ce zW|Ky=w@a-(M&@5$c8jPP3~{Hc-iHKF6|ju-xc=#2gqr*IDverm){U(lx~bwSh`o4d zj(Gd`Rp{_XU35CS%tU3bAl5@J^}!&`tO^U?vv;)C5o^^X#6DkOLXX;XM^(EtKr(xB%<37tK$v_@w;{h|P$%H7?-&!^O ziBf5w_1&|}C-*GH`ez}{^u^d4x~6*C**CQ}9^@*{ua2E6T&H{ThVaec0AAI$ouKB* zUmi5>X#brU19_OmZLI(^0xoda?j)*;miu3{9UPm3v?gzq8=#%FZ5W;;7YG6$wmm7^ zt$OQr)2B+m9p{2_v*apD-QW$P?WX#qx1|w`KFO+PZeTLM&3Y+%H#rMkVqZ1fgUBD6Oa-wMLFwrZQQ3_W+F{6f{CB=G zzRq+((z8VTX}LmEo9?Ta+$VoR1Qd@l#j@D#1$O0sdMWprxao{z7o9&=`;iZKVpZY` zg`I1v_k72Hvz(mxIX`5tYI@iZ$@=$zlNJYty=hWkG2dWESm)#7LyfN?L2wGL`4!YI zN_KzfXl!+bg#4?=U_w>ekHp7wYPnfx~rD``vQJ)X>Q7asA>p;iAr+>2V0zZ_@Xe$jn~kt*Ema!zZ4<+r4V) zrj5107Za}xUc#>3xh2G;lGh(NUDQ@qZ=C>^<1Szing#AdWlzJcARVPK0y~`>pP{8XB(hRIlo$+KbO?Z zPdZ$(IuXRm`FN|HiW-_=&VHM|+YpACew?FPMwg^Md zzDAVwvk=~*zZinH=*A5HRHtn%)BAR&EJvvtBw)TMf^RiSMv_iN71`6j)x>Ro4{cCQ zjEl`!9r3w@Hw#IJcl{Fu8V=B-LRZF!vTcg2m?h3b=okIZ*W^Cvq7G@B0yKNHYA`S@ zI1in9u$<*)uI zec@#w&S`(@>J z&JSe|M`W_+-Fo9i_RsSO>t3&y#js8ZMx3*?>(ruKy?yVliun+`lU(>~Zn~bCEPbMr z?}ffhF2Fsq?nHp;ncutoU{zc0xloGPWRw9fTdN{VL2Zrmm9B(4<5vcM?6BZ?c0b-Q z+}(f9VN+n=FO%N-IiGcrG>+m=J??O-LX+Uvm1>>AY0%c7%)9!2LvybUF#XKv&i9Sq z|F9117Lu~Vjz_^k0|~VGL1=_#rFI+)$bzIF5O~5|xo&51{U_6nr7Q&Qt0y&W|2h5; zl>14QQh9~qdmY>jVOVr#b&27_1=y<9=Z{(%K-%)$UR$s`vsYv5^e=9tskOVSa^q8XDteb zxQzt%fxDMj^guOH3bhJKDJ;x4CaN<%g2T(~d`lfiert;f7>tY`IDYg<;HPKgK1~9w zi2l&AfQk4^OS~kiu==GzA92WWKM4W4oHVn&K8cBhAkAFIWo( z#*3q(UWL$^P*O0&>)I^N19jY6+gwWY&*iV=s}2m%LOs)dcq!*>}gpfwq3Xu zg&K66V7#2D-<*vtm*x3Ps63Z_{rKvVZQP!IpM>S&`$x%W+xy8aqia&H+^%JLETH7? z$nQwMc*Lu812_q@?bCK8Lp)Q@dLffZ;sygFIFqx7_7IgRfavUFM8n6B8JpfU>K}Fi zm_`jnWG@QVrv`iRnzibxiOmLmOn5;q_;aOH4pV9h2T#O;bCwbMMYtG4uP=?LkcWY! zrs4#F7&RuM*Y&LnMxdlc0`^awN~Sm{u9PDBsgi_#53I`}@yKrFYbhFiM}c8nz~o;b z^bT0p6|X(2OlSxP8UNQ{(OLx90-CY+%?n}}|Lg33ZQ&qQM-VZwlr=*b@B#JGe*oV* zJ02Jpqo?-YXB*1M#4M~Sas}o*daFiCgGF(Q4eB5RTrbUpaUQffMOa9DJh~-1p-AAS zVX%SNmq=)Igxbo5L4D5GlzsY$L*VAX9E$)8WVrMz2hV099Ip~5EG1^}2ZZEYkVPbF z^@io5W$nK)IRsc@2aSYA5=yav-n~kqT55r{m=|Zu1qR_CyyP`JX){qw>(x68o;soz7yoZvWW*3zQd#al>Q%Q z$VMysHNxiLAn^4^N3!Y}LFY%LW##ev+|~#;pX>XL=ms4TaY$aMS+OiCKkI@Ev4G@$kpPW)IE* zdAO_S^MVVcOwc00+!eBCUpq&L^Q z`G`E1gomk)ra|c`;M7?Bp|h6TE$xnmDx*P$y+PZID44XD!d4+6ez6fn9H9t)4WK1D zyxh5IHvG#+kMyum74+iH1Xu?8n`?ak$;hZiiofG(Hiz8t$DA zkh8&H9ImHud?4(}nBGxnL}NJipH4NZAsl=QlYG=k(#tPbvxve%bmeQ&++nh8pO&CO zIS#6hFE4)&m-Q&)c0U{VRZbjTUj^L?aY7JWj0G>HA!&f#Z`)Oct0>~BYbhfCgSVi_ z!84!7VBFCW>R2-i;gY1Lgv{Ahj`3-Nh@svcghg5WfHoR4oT;`11aWom>}vh@b{NkCU${zws6uC>U*{9&ms*A8dhBpPy=}U6J=z)YnnY+90DdO$Ea#3ykwuNi)q7=f@4&?Fd?=UfSUzQQ9NjfCH-|MoQzj+k|t=imWt@|aiTR!m~*vUqj3%e5(y?Yqg!8sBgQYY(9W-2X3ZPq zgrlNEi5~>9Ip!d}k0kVQbfFwf2kv@CZ#1V(oLAWb#h%{g;uQ{`q!oy*g2NPjrKqmB zv~<9gEHZ{@$C-$G;z1GY_?j>T)m`{H?x-wS)1$v@E&+$GpGbGb+cUa@|JcF-``I8) zN8PP5RTgN&mP?B-3GH{kXFN<9{|*&M?xh{7WwuyoinBVbrC8j}m z;=)s$UR6IGC~uF|C#o#5sAIr;&$g{2yvkN_kQ-&ws3wdJKksbRxVgyf_Jm59wBPCD z^{-!J6v5bQ#3Eygho`8cya^eNX(G-mo?gy^p3VK~fvp?{$c)ZwBnQ$G6k}1xcN3&F zCx3Wu&Z&&!*$dsWLc{cc#0GJEWPCe2Xw-kVdsK>aE)d8S6>jCPjJ|aU^nu4cx7}_O zm{?W0z-fv|%~Z>r3x!our?UGDLd3_fDUrx(WqAkt^`$C)q*x`E)UU*^#p3x?;E(%N z(?bKP1ditTb_19GRh@{H3_Iy!@uvrLfJZG1*OEai((-z{EVFO( zQpPr?#VHCE(9Lg<@{M#PL>a@#i@ZBax|pEWMQ3wzUS#~IkbhtQ60jNhk)%$9ra$*1 zw8}F3_4F=8x0lXC`4E;lsjta*R%sSe>M)=NJzqAgS!Yj%mp3vKICL{YDcx`#22G9( zS?NceeLV}ZVhIP(@scpRd-9#mjVq4KI`9~*e0EA+aA?3~PD^#~nV1dfP!MqgT-8s# zk2@QjQew4uU`of<2&fawR1dnMp&)7JclMDW4kFUe`$WWRk=FOH#M(E00Of=~b1tCx zN$>6;=WP*Cbh?_Fc2c!zPTtuO>Vy0s(KTI!o`O;%kE-LyFJVf5Kv73YL2&8&JMR%*H0u2`MSEC(D=`&WwpEOo1|6qw!4%cT~~e zrpxEs{qQ~I`ys15?lruxui+4^!+zT_PkXxDr18<8V9o+H)jOeL8Kdr7m(}Ca-O)l- zs91h1sB?5m`#~#Oz>v3U1b)r|g_2LMe%o0iQ>~MAjXb*aL2nKP2HnrB-gZ?`h0)Cf z+j26(QeBHu`HLcR>|JyiLvuDQy!GbQ*sn(zx3f$_AA5sO>SvuYxSXAmvih|RN$Asm zA~ZLqLbxob{a2TQ_A4iYzguYVOs?;OBSvt%j#qFuVdIpYb!SuZlRO=iP>!BpSz&IU z#9$5RDF5&3SH0i9ZR2&|2kUa$+oE>&mZuN4H%Pw^aNa3P+}#+EX#^@mIr^^`K(Y3% zG<9|ViCT}`tOanzxFj^k^7SGbF?>3yFF9$$!7LLPUOaSbJ<(_pAM0PPe;@937}ljE zB-~PXDs4+8tZpwoUbQ_mNC9bGljMwR5{1It@>5#E7s7#`;BNHVuGa-D@lPP@1^sGN ze{-pD7BX9$?HQVEI~U<=TNUVad&OfKh|AfOK_f|y-j^@N!V@{^fv3f?4KHJw27LW@ z8T1khx~oA(;Pq#2Lr0QV*>co;w_)fb8v=KBX1YCkFOsVMAbsJ3r{KX>fm` z^Q<2ehDy#%t3z`bx4e*lN{f+Thgl5O*>pdNj)R4d^2;n4aw+3b-JiQQ@Mb{=i)z8w z`l*?WJ2QqpCqwK?lSiBGnY&zvTQ}Z?|N>-kK`CzARKV#>rvueTbWj)`jA`a=_ z?8}(q?|Q9PmtkHzV?FbrhkCHCuprrr29;F&v{{xCiGSPh%_A#uaps=qH+MicF1rE! zbzbN%W**aNR0IokO;FaQD?y&p9-@OCEOT9>=Q7`aZe;qqkh*&z&n3G&4;sNeY~ntq zaJ%@=Rq0f-j~4pHyYBwnC>i1p}B2CFz0qD#fycwFAkU zAh4ds{M9Tv>XsikY${{`4QnvrN;4@w5@gdxfjkt*wsyI^uZoK0}c>IEg2jVkrZO z&rkhw-6_Qv?tWoZ!dEVxsXq}8uXx|&pn=e?lFEV?&Q%66Mr=iwz7MU#wI@Hva$XN~ zHm!lo3-btRU%P_VeS#~5c|X7aSCRCK*x?v4N9;j|JD1AICBoH&6o?<}`f1>tBh;QO zc;ot4tXH0Vl!wi{45nf@F-UP*N#?)<63amxkO0u>d8RYaPD^k2<56@BS$te*jKZHH ze@zUg7V-e?9$xEFsWPlii%t9*yeB07y~5y?J_jJ3s+=*GkS6Z;wo@>_qL7fjnAtOs;H{wZo%lYBbM-{@ zeU%vj(#spkFS}p8k^7l;>VmZ#Ubb%ATqM>d5Z!z0^_9Gc@|qAd{Q(}LOqVVtuK>7l z$^>|=#QY15%9(Gj`UdTLILU!dN_LJcZ6!P)9#hmSA=L3@P<_5CwHr(Uut6Df%b>Xpnn@s} zxU>JLN@qvi3=&V#i;RGzuqibU)#I6>9p$5o$OJ+rDgWc;o9daj=9t=q;H93Rcp5uR zSbk$i@2zo!9CU$RnphUSWiFv>asHEqKM9=P9M$DM4Q8&}wL|C|wp?1bR~1}k%JlJ> z=S_qYDMx3H42{0i*R@2VTDvm@k^|FX8p#4YV_tn8#H~lhl!emClBBr5!{T;i&6wgJho62N(?aX(S@i9!wL6ccGDDI&K_q zWEMX7oF3?f!mc93Kg9bP3)2vD#-Sl7sxlq3ju6p38&wG5B+{)a6ObS7JV)3P-M6ij zRHP0n~K|I9b6f@+HGT&rQsL34b!|qa24^hp*K~6naT-AF6)_4R5BkR`KDKr~ogA z`+rB}RA8b$SqeDN{D{N7cj^!L=A<|i7Z0LEa9eS=R%xQ=(h`9U8q5HSF628O7+5N97*b_>_>DMaF`$rne6!=)5`m+P2^^=X#-gAp5qdar9kaa)O}> z>PG``HsL;*)%I0oqIcnRQ4|KE3s}*VeX+(%FP|~pr`Eg$9jFC8I$8FmiJ$o${Osft zu#g^%o<%B_^+PZIg-5`}C(wyKFyDeAtx$mN8uO^^Oddp=%aeKtH)INCORk(pQ$`Rz z3f4BSR0$Z((e*edQvr_FC=f0o9x}wCtMXffWDakZ+ZWB;crYVKpMF$J<9iUaX(Y}o ze_R1p;vEJEO^Fba3f8tgma~0dxH`PJerT(&bO|74fk22wEOf@>+d{{fSqh^50z!k% zxS>+OTTW>nf_P=6h&@CXZGP9*!kUX=M+Kd1{}!N5*xq6xWNk;2@x}GFiK|Jsw#s^9&vGeEpy z=A&K&mjXuCNo$oBfowi+iO#WTT$iWFCISJXyJe9pqzexawkWWa2%M@b>ysQPz&g@m zVa*v&$Bz64AzZVKB(9jq=`$hN)k@88b2J}Tfkn8n?nF-kdGGh!`5-05XL#aNAyp?x z>~jJDq&N=0_I^8)L6yg{bp#9HMzUfJLY(aOC-~lmLQ9?CTc~dph$GSBbsBOPvLEik zScJcG+j2C;qSE}A(Oo;-W~h9-G;mHc-n5W^Kg*p*HtVYTqvqB^a$d>Z4br1_oj|+` z)l&7Yw;CmD$poeDenB$mE`UE^eap}g*D2T_wYPgy*4Yn%o|FGk|E(fBC__l*=WRUp ziOL;KSbW{fTj2q~(pM}uzd31+CZQs??K^dMEH*LSY(eIgUeX7n71e8*$*mil>y$6l1j3y(# zO1cvUsegWy*4>N~UtbkBPK{HoW9#%_p=!lGaYO9T^nyIETJMqG`TRbi6+!fLSB==I z@^Dkk_DaX0zW!j;`P>HlM5J__H>am|IBJjk1D`ib8V7F44_02Lv}$+4_kQ?3?oK1W z=P5o>Ut9LXz8d~JHU2?su?y^41{V+3Y*UI^zBD-u?jPV>^2445LUo?p_LOR^?QZ-*QK8Ul10%CZj`;suBrOT4lg#M$4Dgr-)krs_5a*3p3a=M)_R zEZ>neF0_W+f-L3)NF}2UX-taN5q_+V@}JWM@NwG?$samiO9l3HxgfTq;0w}WSnzrF zMXQeLikY7rEJuyN20z9buTNHdeIj-d3eNY2+<{Xm?m`{ao0`t3)u#~b2R)|&6Z)IY zvnPJ>_}+SxsAU4yH9X4xR#fusAS%1V(*h~$XjuHpv(GwmR2PMvDOm|{1PaLn97AW+ zr(Jjw;3O;S*s%5CzlfmbCEaiyLOR+rdoHVuJWc)VxbXs*Z$pFVglFfS&;>N{)D;!T zbfPL}qZ@Uuyr+2z(mIbQVo49N#F^dEgo=zL^9b%8f%^!z^f5~WUjZ8eSL%t>_34VQ zF9nlD6_Po`l9kkDl1PshC6b3H4=N6nHvhS9xBa!@M3iO3P6X=Q&t)qC}$mLOjI7CKmBv_%C>3 zQ?(NThDpJz{@sL$iR=drXH08yW|Ir_&soHXGZ-7tMBx5+6MQ9AW$-U<=+FI&n=1qR zATT)t6*F^`|C*z4WzZ4_x-hfJ8M^*Eiht>HhR=4666Grpq%}H&{s1?HBTj`V^M8N; tdr3enf1lt=s1iBLeWR-X{~z@;j|==^1poj5AY({UO#lFTCIA3{ga82g0001h=l}q9FaQARU;qF* zm;eA5aGbhPJOBUy32;bRa{vG?BLDy{BLR4&KXw2B00(qQO+^Rk1{(xBA47DH#Q*>e zX-PyuRCwC$TWfGs=Xrjf_nci_KuD~_#de@V627#ZsXJ|IL$Pj|=`_xSjAL4i<6=rD zNgHR{#%ZRLaWgF!8{-5VgYDFv_<~KG*5kI*bUN+C30l#P?KDgrj|Uur3LPYj;rJU_26`P$iAL7`n~}GJOh$R z4}*i5!qdwEo<{^u;SL~?ASx?8NI=!K-`BKMz)){z<=D8pN5Y3m_y8aQk^q1tA<5 z1vsa<0Fv)>nhNJoS?=JJ_JFEN@9Cl8>|z&SxgdsiRcLEENqM775CBRL2_peB24;N4 z^SOK4f?!`y8#5VM;8gUmW~6fxm=nUeoHP=^+&=m~<@=`k=Cr$-@20W`y!X?#>L5^s z=jE*GoRCZ!^Po017UW?cPC`Y(2mpbTDw5tv-J-71X{&&~p0$%1p5!QGYGWOjCbda!5Fj97;Cc7Lt4#va_qz% z4^+(mXH6>wghJT=LL4V2>&#~kcT@wIN8;41U+Oydfrr59gYC`E>1!l!U=GDaf_c^# zw+{uv3qd++#=n@|l-_%x`<&e0N;BSU#;*csCGmzCKQ|>n@ZGmtXH{2wbw zyv#`D;$C^03&4$!yWyHy^9CyBy`+mMV2K%@Hsei&R7z}{7RAu}89jTJT5WCO!vJ13 z9K{0wmz$G? z+bFyvBN3j93Dqr1PYUpAYW!OW>|lxTlJ)}p_5=^;JJ8ku;1=`vYb0)P4)p*`5&e`<8@Kl z0qjfo_L;yT*-hy;VySR8t190^s)%pr#!M${O(yLegqKsz>G7!;4hQxuWz9@~ ziGY{Pcuh&9?D35E&TdM-UG93zwG_&>#%4ID$DH)3{zL6kk~Gbj;s2KB@wpP=A?Zg( zuDOu#VR|maRynwB0I=OTtUdisdvdA>e)!F0c4B|)vV_l<%(ys;JCa`aeOsMsPVc+e zJaLvE1@Ij?jYgC(L((HoT6N}d$CU7dM1r??#^5Nz z@2zd@>h0@kvuxIV*$n@ZNbsLrpl8qZc2-X|1OPy(ru0}cVUIz0Aj-B#);p&s$Hv{= z{fF8U|5t+d?!ApC_qQ%3v9pBUI!WE0$CdTX=^o@d#Es73izMfftfBGtAkat4D}WO^ zEChg>g}VX)+aztyzk&)|R5JiP>Ks-K9qXL5C!9La#(~Skkcy-&^*lar#>OH`IlXG+ z+S!ZJ$0FWP?|DoXSH7sfkOve!RJ6{Vv$%VtqQcuG;Xwey5l@&S>2F6z-JNF+cU-i! z)_<^l4uSif)Aqiew&kY}wtJJ!^T+nxwv@n5a)Ra4Abu~^oIY42lrd8xMjR{6o$DDd z+A6MDuq$iEhaqh(@&t00r0s#z$3~8Sq^xBJk9HEk=V{z5;XVL6T%gbP^|W17an)7c ztqI?Du+;L>YXH_Zv~-`C2x4KGUA!;0e_)xc02Lop3@@l}P7ha9cg)|aT!h9F# znURs;j-lSpSgQtEQ9v62FTAD>#6u+S9_sD1c(lKb@4nr-ESa=#c^+FzEdM5656a-J&k%oVG~nk0`Dd z(w1{W+0~~Gv{{S2p<8-Z}!C zr6RI4U%)fN!@-?Hy`2+WJK@`ZIESs#Yd|)59v>mG;`BQ?OgOo})lTkjU6Sx@huH+n zcLDf=hL-N57jBbNF$%S^9#Cp;mGOW=Twb>;D_|3W$5D(4vq3yDKJGqQ>It=tU73W> zEkS^dB_TA2Pm*jhn_Z`ikfAX%@S^ zleEh@tZr!Op6u(d2_R@zCJV%Y1}CYI>g#rmC4AdR;s+&Evd&45kB_@MN%NB*LS~r3HGrIrS=jk%=3NMOB}<;IxUlRozCf3WBBx3 zNq;vw>QD}Br&j>1nX|b2 z_+_<>V{9swQ2;qf4Hks|B+m|xtfKK7B(1#Z5P#Uv(tYyso8~bgzzc)mq`W z1lm;B*wqK&Q2>7}DcZBIle8rW^rxo}wim71N;Rj4XQfs?8w6S=;l75J%d>m|Je(Bc z0p*C$8uSSQb(J##OKr%EgzIEQV{=}Yr(+LEL<0-W}H{<3a z;iVu5v=hLFc{juhAFU$mY7-A5m+qqdgcONuuTTRiE(@rP0^*tf;E@gv4!n97$)^C! zkIa+}93Be-ePQnP-2;>A=FI<|Gvih#JQ-GzlGB?;{^&|bAK&OKlLn<&?b&!nAER+A zfO$n|FTCKK)?ahe?tx2rs8myWsH)0)faLE7fi{@O6{+U*(JN&Cpfr`2%L2adPXIv! z@KFG_M=dWYE8*(^HqO6s_nFIiu9+X&H4fk(!=EcQoRe=QZa|9<0pB^uOp55B!U>;= z#*w6qgy%_YY-s5|gDF1DI49qIngj^Kg)c*5K}6E1b9f{ObpLb^zOSdPQqnyDqA6A+ zdgDXD;~4>m05r$7^B`&~-bj(P6X8Cma^OV( z=7b=e-vCsGfW4r^`jGJQ*q9qEn=kthwI_o>-y!)sm?j@mO`a6#AjuW^MNzTE(CkI& zaS8W3r#JD7HQponXO$K9=A;ol}!qstgTpg1c Z{{!~PeOa9a)Aj%W002ovPDHLkV1jZT)9nBN diff --git a/resources/img/circle-16.png b/resources/img/circle-16.png new file mode 100644 index 0000000000000000000000000000000000000000..9984541714ea2e78e5bcd3c480ee3aa3f5be7ca1 GIT binary patch literal 389 zcmV;00eb$4P) zK~y-6ozuZCLs1Y0;P2kBs0Yy4X(jbN+v_h|m;SmbKAM*N2FBYAlS1uY(M`xI*APoxmk_kWFDK;3b^l6$7kSay=Yi zj3HW61gY~mZqTWAs7=^SHN$d}GO?FB&ji;<-q5cSTw(AVxI%J>Q7OO*Ce=63LC{N> zc8*ni;J#6&=D16kPM#$CPh^?i?Su zrjT)~ZlqI%NoJBH@GXv2jg*qDL^4+Byiv(*;wW`KQMYn!nfe$hbS{@^f(vY`EmQWx jIxqWH<}M;$n(96QSQkD23raJV00000NkvXXu0mjf2D772 literal 0 HcmV?d00001 diff --git a/resources/img/circle-32.png b/resources/img/circle-32.png new file mode 100644 index 0000000000000000000000000000000000000000..6023c0e97862d2eacd2b0fa78dad9ad5653301eb GIT binary patch literal 724 zcmV;_0xSKAP)%_D0Ty}iH~|ZU5IZgcI|x=F zfhZ3G7k~{&F?mRV0}#u_VruM4XFQn+)1swTX}Y`W|E8xOHBLnMpJgDggL7^O7zNG% z=e4;7>;fAi@+((B5jnB~RbUSI0JH(=Q`^8IFbh;O#XdrK0JsbM>?63NHL0%!OabL% z0RX3g1uy6la38n~oG@c6+V((a$@;tnYPkS_OTaf1WEHrPWlGh*4y+mfufRwq0HEfH zzW_>E#Z4RsJO(-j;?PVaee-zQJ3u3^a01AbfoS?`!XJFky!n%V6*qC8nYfKa0NrZw zYF|0dw5I@+fOWGws!0HIhBEVl%K#o+MzD;1#|K_;>3`iB20p64T zYqk%(1E%d(ORG)=G8fIBMuD>&{5jnr`Z@ibD_&lJqRU@oF-b>-c!^&C00002ElnGxQj3v*5<-m8t+phMQIZ;6xOam<)VMGb zKuz2jU!*S3s)<3xM|2}dqCzxjt8PTz1}a6M+}?gJ=3F>;?wq-&_s%(|H~3E`xidNU zoB#Kp$2s%Oj1m#9Wz5Q&K-S1q0HsuuQtF0GIV3#SHKMgyt z0h>kSTwKm2sG`+Z0)J4xwT=UC1CIgAf!Pk5vw?Qtao`>Fq*%`W@4#KD4gugPV8THv z59|RR1nQ~~xjwM5C$Je+-zDJ5#03Bp@PdQLRM6VXe2h<{?pEO4ppK~ew#}k_#BV+Nb!uyzuBrq8=3;-jsiD_iu1rtA~IGeFnhp#ma;Eu&3OP3$)Q7MVblR@!v&_Z zJr-Nvc;uBfzP6OD3TI3KS}lM0!5f|QallemUctq2CwKtEe!{AyJE%Yvl7Ha-op9B7o>mLbJ=yPmowzFt_2y!x?i395eE_Dy3$7 zr<4+uQjNe;qxfhzV+t^A(-6Y`cWGr-PY$FhW1&E90D4%cGAK5 zI^Y)z@4b=#aTx0aE?bQ4$l6f{XWJ2;9F8-Eu2p9vxa&Y!pNyNKglKx_emN(Lgj*95RJ zL+AS=T3&!nv20AEvTYF+3Hn&;^(i{ui2e9ZAw3t{)-)^L<*4*9usQ|j*F?PP3TfxG z>{UQutE2KM;60!%0ms|X7bjdtWn;H|#R3Fm1|lZLLdpXl01pB6RXE)MJPdq@`>^!I zxMSXn00Fr!^f|mJQiXIX=oOoQ4jf;S!PxEyp4}b%x19YEV$AkYsoDz@&A=Am8RB1b z$fE-lr@0!1ZbGl!7Zd;DMGn{rY!Q*lH*h?S&raYo%D0xUv=`{`!rh0FkPzMI@oGC@ zS#8Rp=arQ{IVW{(t_6J{sTX~@_Gt4QPGe3Bdr2sPl*ZlVQe^mr{58M2! Qvj6}907*qoM6N<$f_{k1&;S4c literal 0 HcmV?d00001 diff --git a/resources/img/clipboarddata-16.png b/resources/img/clipboarddata-16.png new file mode 100644 index 0000000000000000000000000000000000000000..6d615ef77b2fba225855802cb3e4f5d4e1b1ab23 GIT binary patch literal 314 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`oCO|{#X#yh2s3WX6WN&PEET=(wkgV~9oX+sTGpOojrk_gnTh zh;3?(a}?0k&3&`Cze3(SwO_G(>{F*T`lWsWGT58Z!e{`8q-zi8DBfr)$< z%)f+%e(y~Rtz{@UAo1bA!2|7cs$UwO&3?c$r=jzp=993M9$E%WSqr#BW^!sR5YKAc zxZWf0GSi+_?;0{6uRH(!ql5LlySoqhKhADEB=nH`x#D)7FFZRKy65|U5LbA8#OUj; zn^T1zAGxD<#UxiIFuBa}QmL3AJI{{WR^sjni&tIuxWgbjr+2U2RXanVml!-<{an^L HB{Ts5PGNe5 literal 0 HcmV?d00001 diff --git a/resources/img/clipboarddata-32.png b/resources/img/clipboarddata-32.png new file mode 100644 index 0000000000000000000000000000000000000000..a4268788d8922fd9d58f18a10f42b526d3681169 GIT binary patch literal 543 zcmV+)0^t3LP)h)4k#HJ?|ZI*!-!DiJG)!Jswy9p1J4Z-bJuF==JokhU=iFtG|O@{>zfH1KFY h4Q)vLIXc~*e*t55xZec^9X$X5002ovPDHLkV1mug>2&}A literal 0 HcmV?d00001 diff --git a/resources/img/clipboarddata-64.png b/resources/img/clipboarddata-64.png new file mode 100644 index 0000000000000000000000000000000000000000..9d945d01d1ba90f1d95bdec111e91d4d91aec7e8 GIT binary patch literal 989 zcmV<310wv1P)$Z zybZ`3?O6&JkqXe2Qp~)n{`O%{EL7Ed0S1AWz#^b2%Odas=+DH~44l8^!Rbcj=DH@p zJo&7g|5pZl+U#u-6cGU~`jXcK7mO}1kTlDnh*W{|z-eFy^^@vP;B{gJ0eY#QNLPSQ zz!Tt(s(wr?Jz=OJ~X?pgsbF-LhxFluxi1HMMuItXmEhhqeMpq}-jje-SWXGq`im_AcZJ~=qw zMt%5C#`HT$5EZ_ip8-z9=$nf4s{w6UtO8xshyP5pld3)wk%Pcl;4rYwK}Q^Bfj7Wi zRs9ju{u$^A{q8Wf&e;feAs{aMip`=_^*eCeL3SF%*1J> z9QFgcX|d}|upejhZvP7C42|nFbv9#%EOw2%v)B-6N&rDLEm=CDgud_q0}kVsfOcO5 z6n+lyMu4iW0K=BuCs@6KI%eu@Fl=;rf!9e;Rfk37IgXQ{s{Rs@qvS9NdS3;MfzC!6 zLLU0Irxgl62b6h0nFo}4K(SHg0lox5nFj!69Vfzg<{`+1OHoWB>-^wdd?NhWNSR)KfaADD#T#O4)T_#aT0 z3lz2)Qz-MghJDty)@8)mp9BO{fXC~=*=Tu0f?Ffb1cXSqO6w~#Yf(~NnMp)Ih=?k1 zgWwqbT9UC0yf?b{r12rut+6o7Rkkf>f+;dNSM_0UH&Ce;xh{xftxFvSZp(00000 LNkvXXu0mjfL+`Wz literal 0 HcmV?d00001 diff --git a/resources/img/done.png b/resources/img/done.png deleted file mode 100644 index b70d72805599e17c091d6b170b3dce3c55885407..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3543 zcmV;|4Jh)7P)z@;j|==^1poj5AY({UO#lFTCIA3{ga82g0001h=l}q9FaQARU;qF* zm;eA5aGbhPJOBUy32;bRa{vG?BLDy{BLR4&KXw2B00(qQO+^Rk1{(zg7=#R3Z2$lb zElET{RCwC$TWfGs=Xrjf@0^uDR1O6jV+$k^Ogx=-rtY+ffii6;Grj1FUDMh)E>H>x z5KbB=`O$GREf*W(1UW{4cEOk6IHdKs85`~5PTVFKatzaAV5i)=aCLE{*I!46>( z4sz$S{_5rb(;;N#-ypyw44iQ;WQCWfY}J@lK>MlAarT1u0EAD1a0|i-0ss<(AnOZ4 z3JTN#K)!nDm><#bTfKGLKa6{ZJDL@soc#=htpKW_u+=!?aq-aVUJ)k&Wa6G-MzsR0 zrU0nA!fC2uHLl#nQSAZFc=zdie{PmMwFfTEz`6ZnNkgWV{BsZh0}?d!X!t4}^PgiD z1Y3@F&}OZX6<7d00w4*iqFUq$AbmZM{-0xHT%&YD(q4~uquB#m4zH-Oc~#T#tttto z%3%T2ylg&jYy#H57GQ*&DrcZ~TF>B^RY1#;j(P}>f|w3Jt^t5Z76Do+hBx~0kG=vR zOYi`Qwdwlgu`#EB*1lDgwO=H-=}Oi@hZ~*o?!zxl?$)K=>t6dkNb?a)QdOz`+D@#l z0dzCSpY`6l{rolG03XUgiU1aA+5oQjM_#lWXhmV*Kvl%PHu^TqV^o_!a4PV=D^ zBzXr7zXIS7K)gZ2)z{nu($kX%o8DQz*sV!Sq#@*ows64nF~N{f5f2L2t+HOfTa!3l z!J}pacpAWO1Pc6=ghgW_8M>13$8Rn+RUaGoNdPaBoEF(Sk6hFP+TLAFoAo#hHf_j1 z!#mg($Tb5_qC#o{nxSbVApml~a0s}pP`JQ==K=CrO^NqI>1jO`0bZiE`c445qQVQ> zNAX*X2}?HuxRZvz4f1wbO(tL@1vHd`G^99E^wr`8TQ$gA{hcdF5J1CxQH4X!Z3Akn z;`zdr0;xp^5d;%_`Z0v3)Ah;YdJw-yaKdQLlLYN1*pY5X#-_Q=2UpCc!{}ySh#}+XG-M2oFJm z#eOEtF!ZNuW`lT+C=nil_LE%FTi5e?2@z^P)#=!*_ay+Ih|Dzz;Kzc_0k8y1!cu5r z1{P5an~P>)3^sPa>}f-D#1%FqC_J~1L!OSADrapt2WWbGnW1a$gz(Q1z!QLdB$ub_ zldlhBg5E%W63MwSi_gR0d1dWxfB}*f{m(B%71B`nHMqjDA1UM~FGZgvg0xlS z(04hcRv8YdSn@v??jw+xIBxKzNn5;<%1)It7r?W_&|Z3r4ws~-C*O#r!pht6aLuG* zSQ;9?28%9}aLxmGF%!6u21eCX&w_J`XGQkk_JOcm>}qJNekOfj1EsQ33F#) z`TB_E|C`1v>YdiJzakI9a6;*UcnZRI06YrdhCm6EByG1ji`HW+cc<%u!y z$vH6G9TDEQyld|5t?N17c6?Q}m-X(DRjpIy{4`yk{CODRpOT=_@DjnGBzJvF_+}?5qfI{eV?wmH{Fmt z008pT2&*p;yqRDv$!2+Ks$2mCN^CuSSh>>?wUhh77)G}D-RFXlEIRRGD@`hl@ITYTLbnthls>4s## zQ|+#UMVFv~kP@cK>KSh!cX#`#&RDC;vxM}A0G!~Si69;&cz^q;HL=^U>Fs4~I2FYt{nTT;d7jBuU%6!F=2N z_lILb@3bB(X|tqXgcSENL7VgEvx`~|cf=`SwsFj}G@KFH^WTTC^kT)3SZrf?CKz*( zEMLUzIq!~7tOc-1C?QKz0X&oM&)w60YRxd$Hm>fHQ_*@{q-g41;+sjk7<{@kZur-1SzYb!X&F23+ z>IuDdJvq8&vpj7m*+S$^B)7}zZ`x0;HUNH~hTk-fnN_+XeHXxzOk?Uq1=W%m1;~`~ z00NisRwmP!I_Hj0{5^o~fIb(LFh!nH-e7(~`^io>QUV{K;bvHM0T>8fnoM$|&Dt;1 z<{NaN)xTQ3;bYyk12$n6s)ZM9kN?x-hZ z8dHNHACc8#;d}$YBmnx0iScQ(6_WKR(K3d}4ot1cSeEohyfTu2GEu-G80rLBFd zN<1OckQ@N9R?r&j*&j+u0r=}oW9rnEB#kj=?4@LbENA)LxeM(u*bLy765%~PBdd+Q z(|Udm;Ry);AShh3p9pBPJp0+!zLh0iTj}~_fAyrAXO*|}WYu}OmJhK$RccZn16UE@ zF?Tk*7(}YX@{%V{TVVC2y)$>54M5HU_!@vm1?5AYFrMT(S#`D?DP@1(u6;tUeBvo@ z@5grtS3X!d4>0t5szw65{@gt@+(hu^kmWsDeM6qs@4ap3*(lN{YO6N^_;Lj4YXNKo z@V708JFZqe!In$$hbKe%pew0Vd&Muey}Oz%hdUOL+!g`6JglAs@cCC}?Tmdt|I(x_ z9>Fd0^hm@L5+J?+;ErEjEvbwL#NuT@Z|^?+Iohn(PQy)M;RSitv(E!qzxU%iF4~-s zZbC^=ehlVx)>shvQEZ?I%|=dwTFbk|_WsXb-C=W%c>Yj2)G9 zX5gh8yDiBsSZog}OOo^^$)8*u@ndToWy0Waz$5@;v$lm~7l4~fgcrUct97s3w)6C* ztSMce{LmTiK1}j2t!LNMa6!60*>{yZz#E49WuVn%!bCyP0BQozJlyj58i);hXYM?6 zIjh<;bz2s|zXbnRal6YM26|;GLN2T%1BE=K8WtgbgN5kZ6U^tkf&p|K!*%aM+8f*t^nC?&#-tsuF^HD@3k1~4;rFA3dFmD2^l zw)U;8_1|-2=#_Rsg2>8FJ5|oBu?k4nC(pDT>F5RV>6orflAI%}T@VrhJ`8aMmB5QZ z(7b`6_Qv8*S0ro0_K+Wfi@{03aX4BPI)oJ;<_xSne;qOa#sz>qJz{-8c+tC%KNpVy z0O*>eJlhQ5Pa-kY0GNl5f}CoEbE@#3OMa{l5lr^h^^{MB($jjfuy{z)oA}il?-Tr- zsdnBRv3#7Fk=pN6JMFUC4dB#AqyK9*egWWn09u_IH?wEzHd`@eY(2he96bFFfZvsL zyR2%8-zzC92tYBmve@6Nm?!v+k`5p%>-YVZq^yNQzX!-~h#YDsRcYIB>?ZNQN#^MA z5P%=hG5dP!wtsNxoH;a_4=krV4K-~Sewb+dhNm#spc9~Xp?RwC8y`c!22E4~l(#Sm z1Dm(u{Jo&0;)>*{_*qtPs$DqaUF6POh+RDmK|?q-3H08&UAR83kL#n7@qeJi7TV+% RKqUYG002ovPDHLkV1h&PnS1~M diff --git a/resources/img/download-16.png b/resources/img/download-16.png new file mode 100644 index 0000000000000000000000000000000000000000..620f8687f8c8303c97a8af259cc528a254da2195 GIT binary patch literal 345 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`oCO|{#X#yh2s3WX6WN&PEET=$WUBV~9oX-OGl0M-l~CKiqdV zadDjDurxh@Ea`AHAO%wUI`zFWk8|S>2{d`uN zpB9%E6v}omV13G@w<^_pEtG4y&Ln<1FZ+k(f1tw2nn|}+xMTA-d#;Q&4MeC&2 zy@F=%Gs$e?I}Qg=+NyF{wl!^{*vB66H%Su5SE!vc=$hO1xohdFR~&C1?^(5Fv7qG< zPB;6feuV*{b6uqlCx6cgy?W@H`90f&KibP0l+XkKMly-e literal 0 HcmV?d00001 diff --git a/resources/img/download-32.png b/resources/img/download-32.png new file mode 100644 index 0000000000000000000000000000000000000000..840c82bc7d42bb4a1a21d5d9c971a6efebc4af80 GIT binary patch literal 571 zcmV-B0>u4^P)ruf3NrTnac=wmEFhr68~fmJlit{Lrra3?v;gaYh)!QXK)wq+&yri=gG5pe7zW+~ z5wK@wbCM$9J22p&i@=DBSAmZ%{z-g0{*p8X^Z@6;4`9X2E)%x^cm}L_D=GPFHy;Im zBJ^G0g~PT3*TMWI(3ZFbz`O@O2KIqvpxMJjz{cOm9WD?MV8qa&>;J<3FNHH@z14h$u z2|538J&aEHXapP*7YM+sf56r4#RsLJtP@JwChlCP#697pM&vEPC!iJBH?!LX21n%` za7J>ctJW`op|rc83z!0WiO+g1>>StzzL?ow8DA*ie_P#8!42$Q6^|nLNJIbt002ov JPDHLkV1j-8_PPK7 literal 0 HcmV?d00001 diff --git a/resources/img/download-64.png b/resources/img/download-64.png new file mode 100644 index 0000000000000000000000000000000000000000..67aa68df0ef30c4a0232edecfec0c03cb7d48807 GIT binary patch literal 1006 zcmVt}j3n}r5-ajiCL?CXv3M`u?wse&y?3rTuRQ-ci}&1f&+|Y3=cDI&-uD$%<$uNq zZvx(k0(cYfhMNE)(kdcnMdXTztZ?Juu?UJtGw>8>O>rF++4gyKs$em2WSFT zdl2ywV2t5w9(ESQMKB5w1@I=|jVORO0dGVBya{+C3gAt^8&Lpn0^W!McoXnO6u_H+ zH=+RE1iTRi@Fw7mN(B&+u_AIrL~e-4UJ+@kDDOHExh5i~MPzCfIe$Y{D>L96pmlZ; zXfooKrs5Ph5y;t@ihBTzsETvdQ(#T%^Jd_D*^wY3oxt&wdm%8ZOg;-$3-H45l@cLP zJiuq*Z3%8Bsuo}u@X7EO5+S4beqg7n_LQfXLX~SkGG`Y8?-@A96}SwHBx4N#_66SU z1GZMqw`7qMfR@CXONg+{5&+m=qOC{jpL`v_ZD3+nTtax8s$Lo--vZzOP`@>9qtq~W}<+? zu_p8a8!Kw#tdW1c6&u5fN%4Oiw+!OKto5gxUe4(J%)%Ozpy? z9zDQhwQiF$PE-ddf0fl}b}CqBh5%!+FE9T!dz18jF#!@0+JKi0P()dK3$$m}nB#I` z616Wm`x~I3B6b~^IaqzU|1~Be^MJL$0$>V_xrcZMJO{3->cc!QxcqIdkz@;j|==^1poj5AY({UO#lFTCIA3{ga82g0001h=l}q9FaQARU;qF* zm;eA5aGbhPJOBUy32;bRa{vG?BLDy{BLR4&KXw2B00(qQO+^Rk1{(zgE=#u0^Z)<~ zzez+vRCwC$n|q90RUOAazjN>GtL;8!w%rzlfPlnk6r)&L2pTjdieiu&C}3Ml02QJF z5@OJ3X`wCHA}tS5pq7Uy!54;TKt+k+B>@Sc{6PbeXSciCeedk<%st0H&YhhzXYSpd zT`1j^o@CQI_j!J=@9+Ejol`hC4vzQqkhw6)9*{6T4uA|W=uBS6=XLzs0Chk^e6G=+ zntKWufrc?S*TC@tVM(}H8WDS}V?o3?myMOpgz%(-dz)c6=^No}RzLyHF>oVLZ&Iu_ z)v<;%fwzDMlfI#5vjQ|MioN;mjCyUOfjQ>^jd1%g923E5u~B2Mx5oyIl;hqw=TwXr zk%EUqcy!K!U=PSRDD6vMj(Nlbz|!^5Vt~n(FCQGU(gV7nsR;E!Y*lRoUrLi3^DJE# z&s&43hkOf+&RGTYz~Kh25jde+h%J*r3-2&^{;MJLY|A2yEA4mje zDmdFUhNPq_Nw(g}3OdK(%DPDi&#cV?@(!e8nMSG{fF__rplbmP9Z&_dg6W0(HJl)D zY>e_o$}UdQIS25Kf!8$bn9BNlK@=QUgzJEl1)lAPZ_Nr2+yl)`(5m8ik7=%PyxvuS zqZjlTjI=;sW@-g2Q*gV24?3Z>ZB`JW#V}yK^PL~NLAAp1z{#EeWY` zJ#h*W@qo!SnWB6^F;a4cy)aucrzA z7N`)G3H;g<-WYh^z&G0A@rlWdgABq+2G$FFLSWu3CUC~UuLLg4rj)szusnb}6&&SR z{&|7bS$MW04GzGk1%9dE@VUTX1wN!<0iFUnK`EP6?o)867oodCxS%B#enJ6*aA*N; zVQvVIu^zB1gg*aTV1>U^Duu_uE(L2^;OPn?Gy=gGTn>D|d)Fv7)&`wet%y&j^Wvv1 z7w;-8XPkJJ__~J+?hj)s-!0f$;ng}==hgsk+L&Jo{6@T(2VPKcew!yeK$wTD!19F2 z7X|Jv*jO`o<>&b&$ID#W zz*8Zd*8;nydXGwwd=z*;n`3 z4fnzq0fr#xhAZ~LkaOL1!%&X}#&^vV-T>)@<=$M)K8SkYAAJt->4Nun!JvE30Gv^) z2TW9C^R4x_8Q5$bQlo60aJ_~VBh&DNE|C5idQLqo4=uMp!prlIDOlSIZ#d_2L1mXn zuti<9M~NdaKFtH#AwL8gM!nE<`8FoIPYj_iSl6g=4G zk(rcwUYcy4zgsGW*9IOHSlI@9JUU$P#6VKk29&CyRXiVx3N{(o>Ux4evw`gqoH{y* zWrr-TT&%1oT?Fim;KH67dXi0WLdMqfNBA)htj&4CJHq-3NbDb|V!6$P$L}jf7DgK2 zs#pnm=f#U7xFZjzkJwe}q_iRiK1F<8pjB)u{OpmGFnkka51dp7_XKdXXZgPhoRx!D z((9$Fq9k!{$|QS0qPFHiA%u0n77r7$8g9zNm+}e9YXPn8Gri6?S`2I%f>qtgN;o03 z-2O;UvVQ_=vy(JKm`0~rsZ;`(+Q}VivP#%&D#_Af;ErK9VH}y~ll~J%DQk zR`kXe+6!tgEDNpM`<8zM_*xELo4R7IO(}^UV5aD|HCn*i2;AslLYu%%WAGK<6Pm5} zY>8ll7eWgKZjRu*K|9G472FfTG47N!{2f@6bI^Vg`fd_rQmI*h8KkBh4CdhnBbEzX zNW2Cwuqc9CN5Kri@8>(&p$+oguth^-0GC=_+Se)ASb!w~oFkQPuMIq6U{$-_=upe3 z0zs9Q1(IAwYE(!gjAdYhz?0rfTQ%G`4yXH`&<>*tuGX+gS1xL1fgg)!c>{k_uy$eX z!n;-Oq%kpp0Ww2!k_T0QQ#5=y!3`G~`1vTT9QGJv4)PIPCvd%iku)o6__Kj?XRz=o zHDkcrA@swrQeeBJuq?5jux$*^9`Yv80r>#dV^eEBq3in)aDF@Nomty3ncY??AgTfM z{4oyF4PO@c33eZkbBTX>Rbn}(G_39OzL~XxuQPCQ5uVU+tH9TD(8{iu^0j9K8q3!bcY06r1k!Sh4f%S`0gm>e+1vbU$Zp7^|3LLVZ8c0$ruq%^t zWe{W^oF#B8)o6c11eYwFLU^Z}qm?a$ZjNgQqv1IP&+HHBr&HjCDL1MDoT^|el`&s5 zaF>Rw7DNAZCe;SR!*JyoJQ2Z1Lik17SkqgiGgw@IBCA;9M$GtWhc97G|K|*xuSvNwH*Ld!5As-3tq(yia#RQ~u^A z?HreG37maax_rO!H(#mbcUSnYE^nDNrEhDCXF1?3V1;MRzB*f9R4bqj`g`C{z$pp4 zHbucQ4Lgn1XsLrV!4*t-Gm~-z68|`I$Vtr9lvFCf7%CZzq$8PNx6nt}hO3GjG9B1G z#~IK4?;o`rJ(5#HAJj$gfWYUNBma?V8l>oDffd=54H7CYOUDgds^K~Qzs6pH@6=Bs zmw4pourb)A;4?Avo2%Z?3;YMTq6waEn9hHj>w^Xj9|JzF?MBHaC*CE6UY59osk#s! z2M`$B>@b1dCU27BJjMc*f`3&WUfO&axAqDFybL@RzzfaLGZW4{F*@x;CuX3XxQV=n z1E+`(=id^lXX5Kc?5brUp@r*qG9%}=juEbrO787{Utz7a;2kKb4~~Q5Ju?0W6W+#n TIJ7S^00000NkvXXu0mjfV^C6t diff --git a/resources/img/eye-16.png b/resources/img/eye-16.png new file mode 100644 index 0000000000000000000000000000000000000000..392340e68ffc711ea339434e4587a3cd6441a659 GIT binary patch literal 371 zcmV-(0gV2MP)`@!!F*@$~oT;7+^aR$C!;+2OsF+GS;o29krXq@McgvEUmH_ z)R(Y;SFD!uM|eVWs*qwx(0;_`kZ-+! R5e@(V002ovPDHLkV1nL7os|Fp literal 0 HcmV?d00001 diff --git a/resources/img/eye-32.png b/resources/img/eye-32.png new file mode 100644 index 0000000000000000000000000000000000000000..5531c1a5252f0b50190e6493752d731548c55b43 GIT binary patch literal 733 zcmV<30wVp1P)tiUv|}w+qZfEDP9!ER*jKk=NFwNMI+7Dn8TB7x(ID&yF&OgGodQO% zF^9Y>B0Hp1McX3#<{aO37)~*F(G&~V5(U^@r7HR&+_cQvgVcVqivOO-e@h6VVkfND z9fzamNyUdGNs{Qj{HB*iQf7&c;Fi)6^yTbeo>CewV`@r#FxuJr9N+bk?;Y_Gq<{qq z7o>Ozx90>p8j<~uf(C^ikyP;6fuR)h3iI>Iut%X(iUBWTP6pG8V<~xDhF8#@!Oq2{ z2=gAxbNfKlL>@#=11Vu=g^JjrP-5MgwQd|&=#+7v=*aTrwc^Rxk6mbqmYx{hiU(4A zn^G%Yg(Z>CFiv1!2;plPSzgIRyF7z$nRM4Ozv48ORpY4jb37eu(%Fl}%Ks%s@J!)z zd=1w^2%pA`U5kJUf+k|7Lh6h6i$pI5YP}%cL9F@n@x_{OJ%mX P00000NkvXXu0mjfBB52F literal 0 HcmV?d00001 diff --git a/resources/img/eye-64.png b/resources/img/eye-64.png new file mode 100644 index 0000000000000000000000000000000000000000..7d7fbd2011d74575fe311d18ae099c025e684e35 GIT binary patch literal 1394 zcmV-&1&#WNP)W)xBMP=c00K@UYQl_02L(FLO- zQ_=@}iO7iPrbWGUqav_Oi=Z^kQbN--PlT7%!>og{U6(!kTvp)zV4n7T|L;HlHS^8P zH#;RkHS zw|GBI)6IRtv$q6Dl4Jm8<3?PE@x20k%(i14-p2YgO^@^nYMQ1!Y8ZhBuoY=PwcoH9 zr}dbB4+cIKFXEqm9C%~v!t$O75xoSGBpHbNvM(?)!gU9}#}@o0B)tRKZ^JUIkvI<* zVq66FUM#|jG))gj_{GftTq-1{sxNaC8}K-$V?+xxY$Rsl34GH;%oN|@l0I1in6G-e zZ^c76vkAPx!p<-iooyz3RYGG{Tw%AKm}GO?#9R2ENM(^D{+1W z#sJ)cFDf`K$DkewkPZHWk{~-UN4A|-U`@fsCZ_QrENMc_1at9w3D){55vsn{8ztb+ zVQ7=f=L#X|#bo%`t3=4Khv9`1oV8^*W$Lc(l{FR)M1MpMzPmjg5$AT{NS%lCGCE=e zSXuzI4U?NCm0_(dz}kaX@si^8d>B(AVoeoJjCoj#n-T!m;7AU@PFxT<_=yErt8r>W z-|5(u)AyCgm=g+cj$l@)0NHx_HOF}$ro|A+@E*>=dQU;p#AP{x?Z@EAxRbLP));%c zhP>72_)Lz|AzT?d_zdsO9K6dbeAeaoG`$Rt!A_xdI4TzbnUKVJm~rFh9w` zE8muQG{@(L7UM6@!RkV1cb{e|LOrP(<(kA?=SPqlh>uDJS=5eTc0`;xI9PzSM$y@$ z1aN?GAt@rb9K)L=HyWQt4E|r-8L`aDuIsOfy$l@(K1jGy7J=`?ydrgTupTSi z6^Eq^uLF0=T}S zCBO-Rk|e>|LhcwPl;4JDKkpX?y%)RFG~L!Or1YZz|5rQd|0C2Epl$+nTU&s-3Dj+E z0qQ1Dx3vYRn?T*x7NBkdbz572x(U>6C%pjw0TyUl8PSTrR{#J207*qoM6N<$f}z=i ALjV8( literal 0 HcmV?d00001 diff --git a/resources/img/flag-16.png b/resources/img/flag-16.png new file mode 100644 index 0000000000000000000000000000000000000000..3c2b58c46f67f386e6d384a0849d0e9ad531eed7 GIT binary patch literal 300 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`oCO|{#X#yh2s3WX6WN&PEETXs4%(V~9oX-HV2POo<{a58ks& zYrWagB*G@u=kSp6&6*w4F5Rx+)!!g0D#|KyTaep#9_yR5*odDSOD0`E^ThYwY)`?3 zPF^qNl=mk~3%c&TY;+^B;e6lIXc0lfl`}h5u&X}vUANHoA@gSWnm)#5Heev3N3eZzTh`PJ^aO*atDnm{r-UW|K{#>J literal 0 HcmV?d00001 diff --git a/resources/img/flag-32.png b/resources/img/flag-32.png new file mode 100644 index 0000000000000000000000000000000000000000..531dee24dbddcc7213738008845885a005265531 GIT binary patch literal 475 zcmV<10VMv3P) z5=>$cku?}}J%%@n<<0nZclL?!4h%Ez%{|OFcbE%flK4yY`6kS499RU(U9Qmt&Lq`S zXCu}NmA1n9~HpNhJbTm0C)!$Bt3^b(E!Y>1e^e)B&lDKbn}}pW(GEaSvTI5 zRLx?F2{7mS4`4s5DJH-=NzthQRWs|)6+qGzu;D(ZfWuq?oOXcL*)N;fTCM;}s?aKM z>wfK-*>uPgD#a}go7qI0EL$X3vI3l%*`%b$)W{5_I7Yk3trUCo>6^%M1V^-SuT#_n z?z#bZl2q#gAgH{Kpa3WU3V;IOe*!dI|JF+Y^SzGU<8%f%kkm*G>Loy4`UdU)3)QWZ Rx&;6L002ovPDHLkV1k1O$kYG; literal 0 HcmV?d00001 diff --git a/resources/img/flag-64.png b/resources/img/flag-64.png new file mode 100644 index 0000000000000000000000000000000000000000..64464154e79aa1bbc80c3cfde1b41a0b032b5c7c GIT binary patch literal 800 zcmV+*1K<3KP)Smrg}2EF`RgjcqzbD-l#8f>t6}q!QGq zD5#{fvR4p6C1_zIB>0G+kv$f>h)(XB%-q>MC!6_Uik-uG+>hOV_s(8XRhF|>SPx*W zRjG+0vIZE)I7=PPsp_|+OjV6qz!2~RXaeeFHQO@{MPE+>fH4Ob+-Z%)UQdb#>`ch% zmA1zwqyX@2oDX6Xl1(tsGhi-Lt%bB3SXZ%s7Z{JOmf!)C%>V4IkFArkfMMWQMgLd8 z+LW4}-_HL?M79C9fR#aL1~{myUsCEW2>|Wi91nqwL8u9gsOsC)+U@~}hyXW%9Tnj- zs`|K2efI!>)4-96{rkYBy2V`s5RqZvTt(|6O9B9N&lGc=3Lg`Z@0n`-PgB4&up!uQh{$tQy&hdL)2XnB^#j<|x2M93 z&>%1xo77JL03YA{TQ2=3&6Ol&Sa|9Q}xqFAV~#) z^#Il?0IUbFRsmo=fVBz$>jA7)09X%TtpdP$0BaQh)&p3p0I(jwS_Odh0M;r1tOu}G z0g#Z>T;$%8Pzr#T!RrsI`Z>oi7D@qd9(YRYAKCZ7(X12oSd#kmY5R%p*={_(+Djz^ e=oc;fZ-L*$`ho>)%P__O0000 zq2ZN#by_%Y9bPYXU|`P7oSEl5&p9GO+wd53)&2pNQEt?ZIs6VI46%B8(<#07+^AE9ZaNO@H-LRug)nr%I>FlK8}|x-m9*2&@@=a{9aEUY94tC*^adib2S(IR*SS6FBQ z2MPZId$-8=BGCliqtwaiB=bzZdDjoA@uzaYw)&s6CM;(6B|LTgjDIF4Yni+Ezt8mH zuf{yX%Ov>{Z`<_^ynh;%Qc{^gMEuCuL)?jouWfz*06^nayWlO6ZvX%Q07*qoM6N<$ Eg7A#aUjP6A literal 0 HcmV?d00001 diff --git a/resources/img/gear-32.png b/resources/img/gear-32.png new file mode 100644 index 0000000000000000000000000000000000000000..67300de32f7e208241ca8f76c9d526056736ad3f GIT binary patch literal 993 zcmV<710MW|P)I8XBZA{2u6)c+(2K;^h$ul!6pSM{#^~Tm7zPwv1mj8(@y~Io zh#K7aSonS*s6=Nm>S&w{W1{hqc+179ZRz-y;7?8et85QxWE@!OfR?%W z6~IN?g-^g$pxw#mI=gLUYov;Bz8hCv-P{E_L}V(^tg54d-4AX&t8|4()T@9_U=Q%v zqIhl#C2PjvJTcoxz!t+@GGPjUpMYD`=T?DVzh!a5+aAHG0vCYR8V7(W!2O890ErwX zoam}kM0w>|0|*z170N{shW@PYzwtot1AO&sfS)BVFcFx@IWCzSSx@hs>40I<%< z)&o!7PP9bqS5OexFfb=P1r-WPy);2V<&mzA$bWY7T@iUs>7&5rx?^Zc2TkccFg1`J zGudFt7UuyRjTGWrll6I-XDomg`;Gyh1KDIA4owtS|GQe$!plzE9tF-OQm{56U+3gI zBl1fWHBt$QXfDMT4FSs|04)?-`~=tli1m%YpC%gwW=8DW$qHP7XTUV?zyTAC0c{Bm z@W*hN+WWr>^px0pdhYQK9Jk3=4G>8W@X7(@{Uy+yuobt{y{g4^e8;}6B60#aLw3(U zRMoMdW=@I78B0OM!0rQARP{ZPCL;O!U_Tj+13b(DA*<4xk*HpzqDn;ki>nRa?_k%v zS8gS{DytEL3F$eJst`wkYrw)N5t-oyYc!D08*Y&iHy$J-up<-{K2GgV%K0RI0}d<#?gr8Rg`;b-0vic* zt^@N^_+kL1O+=c>PQ_9(9zU(B9|N5^z;3d8aQk%b~MIud7=i2PPW)`-Y^BC=gX9v6`%B63ufI-{zU6krap1<;)GCNL!jqaApl z0mgP9d>wj009iVkmIy|AYg8bAZ7DiYw3qoa7n*XhDn`FsH2k zQg#~@k=4LVrUtwN905!;{@(^nR@FU)6|Ymt-anme z&kO;J;DB>GfQOBCjmmmpYJ$-M%melq?JgU!DFFP~Q0Ui2yRpF21&rSebQ(C<8*T3{ zV?)_(@K~endA~wc2Z7nRkD4M zBzEf>1kM<)-cRW@8P^f9dc6vq9mER^v)hP|?d#>5;Wj>{^?IkcxEbt!{1Rnsd zJXFS~^jt)-Ge`}%B-a7Bc>9A$z&_wUjELWj!#h0{N>v+=09iu*PdV1WYd>d?Fwv zG%5cuL;mJG2i$Kcv4JL5`efMmZ)>T0VB+P#ZFs=bcMJjM2_Ja3EU`&Hvd&uA$ z045c1Kqv+GPu~WMxgB^h0B~)j0*eDZA2i-`!W#zvjghi14(Pre_+qAg0Hh5&O}#2o ziEe}U`~p3Afx*8blDEs??Ey|R+vfnalifpWKzoD|emwC3sW~6P-wN#TFt$aGo=-f! zkAagzJi(lxx?Fp=S1rJ#5Dy#(Z;$q>*Q@GK9-k+B2)FyB>6m)xoI}Tx%^NjciD+zzvbIuM6n@A}}s@7r-qbYXbm(Dlmjkz|Pw} zl;?!R_+#k3MCy3boJgmC{Sa0C=c|R2XUZCd{xTB!#skrdn z$Eg8d&vU>F2GcSkA~-{d3AiJzWsu@ddiBGQmFusUb-)o#$nV5HhW{qd0li-19YuTI z!<*K?xJG3MIZtXLfX04~Vc=QUp0tZ;8QzTJfJ>nM>6NXdW4!`&EiKr1e6 zOgDa?YEohofL{=Ip)r6<%oZN@hTIlo1a8Tj0?!%>MV$a66`BOx2&}-JGipiMihEaZ z0|kz~5%T66_WUi~#!rji4E&sfsRrE3f6oQu60g3Zsyiq_^>>4+y00XT0H@(X_gzW# zMYxgV2SAHe^_K^Jc^(^&yF7@Qqcz;6#8<28u_k)>RA4FbdB<(Qk-;gDfL=ned%;0q z(FhoSHF0V7beW&Q34qUQ0(`+CG~T-KXf#@XW@6+zx*E6}_?q|kXk6shB_hj`*J&16>A0v77oxHb*kaUWR%j9F1fBuD z9D#W$@JJv9{%xbKHv%(EB^RYRWEZ79xE-P;X>&UHx5T0HFwl`;v;aTBt@wVsQf~-K zeJ~*+?YNI3zF_?QQBqFESzv1@e~8N}or3)`eU+-tDy(>#D%hGXqxhS` zM@VI869fNU5MwVetF-N&PEETXq~5vV~9oX(aVN@PK*L97tEWc zUW=G@;b!*3M4qY3A9w^+@GffI%Bp6j64uhj79-^|q3yrDn%}0gd1YnklRQH$S<}^X z|64jv0z&mB73V#VHeThn7GD3cQGHJM)McVE4_Uc=KGr^YCUZXY>o;#F?mLOIQc5%u z+dPFiD-H;W|30f|XPD=3;V{?tzqZEdo_Vt~R;==!B&zqDy?MP@+?tJk*L75O9;{Ap elK)dbV;k?Mj-N4#f0qMY&fw|l=d#Wzp$Pygd2JQ| literal 0 HcmV?d00001 diff --git a/resources/img/reply-32.png b/resources/img/reply-32.png new file mode 100644 index 0000000000000000000000000000000000000000..193deacfb988ca7da2d88811ee8fd6616922c4d1 GIT binary patch literal 432 zcmV;h0Z;ykP)LBxownu3nX=b;}DH5un+Vi#OxKgxBZ?QLZFqp z@~6}|R=oiiz}ny)sHzXl15Mys(q8&R(IYqjo+Fqq09(Kzun2qtJGL)@KSQ`2E$z?% z9>U&TV0zfsFeXGZwfANwKv-BxBLZgWzZC|Kazkjl4V;GQ2CzC}4w8L8fX6Uh0hlRR z+?I6PANno^K+sn>st_PRoJTay)MWn;l%%A+zd))HRV7L5z&3DZyH`2Dp#EhBYJeKx ahrR(F>X`>a$?AFl0000^6 zZ_$_&m7#35GKa-mRA+IRSyJX(aSt64?%XK}!rEIme2VU9O~_m;@X_Ca^W~EsL%~H) z(hI8<-<-`W`*yDS`=-waM64%E*S+6ty?b)(WS7}9Kk+^A6?!0hV7AHu>jrbrhH{4d zNeu59?x`@|W8CA(w1>IEcnX8@Y{Shn16w~X*v4SIo4urQ-ORhPu}chM8mVgW&a#0q>b-8vD}JGm>nZsI{x z{RH71Pxkh;XRg(&i)Uot`y+@kS7Tp*Lv+{lmQ>dFNom~&<{tQ4BwfK&v!Z^EoW>!x^4e~;s+_wtY25X^pH~OVAPS#FVdQ&MBb@07w`^00000 literal 0 HcmV?d00001 diff --git a/resources/img/settings-16.png b/resources/img/settings-16.png new file mode 100644 index 0000000000000000000000000000000000000000..bb7581d506458a18599594975402ef774d06ca03 GIT binary patch literal 421 zcmV;W0b2fvP)~3* zK|~DW5I@ku9BP?))>i;CI4WIX48L)TA^druwuRS~%={J+bxdLxYq&ra?{VHopdmB= z5sTQz0hYURukf-nmYH-1dJ$X5!cU#KN<=)zI>s^5W7H~^Fk3unM8p-E_*g-^>~m4r zjfv9vya;45ehSt9UEvEZ3qRlU03UFI?|9ooAW$!i&Rn2|uXu;2*uWb%x6lul*o*RY zJ9t)%Y~mNL2MDw=AEl^)pBOF53z_-Q1>Qkqrihrv3a+p{=>G2sJO+OOP)yR=mcdHVa5gi?GU>cEq7RS+3w{In07yO-4;KlxN zR{`Kr%;aES;~DJ1r+Mujo^1j^iU=vCL41X66)Fux#I;gBOyXhO!0)S6{x;r@h;P~_ zex(D|^LStpJ6CF?S&4s8W+34J9?1wF$B8_CEh5fj2A>T=6EWs)Od{zp2N71_5^YJAmNGUyyhid8SY)|0BJRZd{B)p4}y#4|% z=ke``m|c(nuUAmF8lP6WC(w((E54h<{)kw(j{j1$*R@(p(Y)GrH~V7i#SZdy91&wF zrN{8#DgzJUtRfNL#Ob^~f-QM`0zc&OG(MMuH<5q|-};y0pYU^Oevu;Ix>*G~@SXqb zJ%R1ZDnq=K(N5zJ#r3;$1t5Actf+)h971pHp3>$cibPy|f*HKn6jPM|@TB5~3jBzp zIEg>=+I94G13>#x`Z6}<(>@ou*Qb;&;7e@Bdnu*K<&?xSYzm)5M61ofNBN|eD>SW= zr3oJ{GjJpS4y2U&BVsDgcPmN^ZuxJMS&RLqsE&mk7*uquP?XZEIIMVhMBcEuYe|?v zoa#VT3kSPe1tk!}iY^uk?I$$!{O;tHFQWYXo{xyRhBWVJwNJA9Ve9_y_#YNnS=U7p R4GjPQ002ovPDHLkV1lC5YoGuC literal 0 HcmV?d00001 diff --git a/resources/img/settings-64.png b/resources/img/settings-64.png new file mode 100644 index 0000000000000000000000000000000000000000..347d454ca9f7c20728838a6404877bf507b3d2e1 GIT binary patch literal 1489 zcmV;?1upuDP)`A~XgeL@}<6@gqi~)QwSeff_$H zMi%PE7=xlFd;~xE(kL5r!{=1>b=LzCX~yjR^JBwrEAaGs%FG&Vz;(qRwgbNM6jm=H z7gykE@syc06c`PfT?#*sNoosm=YVFQ$r^hAc*CVB$|)UbXPN;Dz>BK7uQ=fQ8C(Wz zw1$rX3srTO#XAMqgh{Az>;gJHCS6M$RMo!tfUg540zE*h^=Jrq4^syWV9p3DfXf|X zma6J!$>*FgQdEM=NuPmtg#PZ^~!mfB-L({7lddw1kef z)0V1UhFdU6t(4)1foFjF(1w;GuOwIsB60<=0Q1eCf+;z713Oi9xJudcfqOB|o*4}v z#GEtMscLsM@`Fd9%<5_ZKBe;Z+62rBvIR0Uz8&aGV0@7d0qw!IKrJC6t(b;tqvxxC z1J`S;%1eSgw#C;_+W5l}xiAfQ*IQfY5(lKcK2ZV@9k9yhZqiMVy9!rZOw0 z85s7E_ZQX*i3>X~QW)CkIPFfbZMr>W{y}sIR2_R37{+vT^Mn%#c#e6>oPi0pf#uBy zf&T~}`63MflgdPY5^&U0=EkyhfQZZlW_xll{)z@6iBJu*}P!`92Od0ON z9^g$?J-KQ^fBLM5K8iaLt^n#{e07+vk}Cl|?4vVRm7s`B!gQQ2b3E?FoIC|{Fmmzi z2WG44U}B}AqwZJ*9r%6PiZl!?^3|8SQdm{DVU~Bn>w<|@3L_eX&D zfvKlqMN9?eL{u|D(v}WyuEXY*|bpT(G{Cd2Iy}*sCI^g+UP>lHsh)4tQ zA#US`F?aoaj%Q%1N4qM^Kdhzy67GHeD$o$)n-6^F;2k8V{r^Nc0QkjW)Kew$L_O}K z=+mK%O636HQim};z_^kxo4bHop8m3=0ZePHv6KVa9WT98)_dmv1$k?tRHC(U8{$cD z0;gigF&4*2L}wDNj{k@|;E;p)h6=*w0TZj8(P_pv@gJjgK-3XFh-n$cu|!orx1~h< zTQIHcIC?Rc6nR4d*a56k)sw9@Yfrnn!bdUp`V!;22w3ajEdXtyGrJGjf0S$ie#Z3L zZ0y4vN+x3(5|bS~>s57m+7JbSBeQpsF1z+q#Vu{{I>2(mYER>j05^p;re+R^obVW? zc&KE27jRW*L()>&rl^QC0#9P@-{!_1odCYZ{IPn2mJ^b5B<>1F9otJV2cG^j+2}ZO r)fNwd+;xRdH_kaA^aMiVtarfwH_=Tv5PQ|!00000NkvXXu0mjfGS|6a literal 0 HcmV?d00001 diff --git a/resources/img/trash-16.png b/resources/img/trash-16.png new file mode 100644 index 0000000000000000000000000000000000000000..43e1f7fee82f83c47d98c64decbdd3c8063a9868 GIT binary patch literal 390 zcmV;10eSw3P)@wb`yx|FqUDvm1Ao99aG$MpCu86R7}a_J-1se)OVt^3<~sRKLgZfu z+WrcKnO(RR&a|Cb9J~V10|cD9T>^KmrTfmD={98m&^c0n+Dq&|2+#_fOR5eh=o6TQ zHB?R9P)%k%fFzmO(jg?>hTv)40Tv1$D3$>54O|^5q3_I2_6Fq{SW_R_sB7RUp0WM! kY-T;+vLS;4ManZK@rxo4-h1hn{Qx1a!yX-CF+9+K7e>he1lvf=3p*K z4qjCPLM}q02BJpX@-Q{hHr+eZGb0j6K~rqkzi+C(-fmGbPjnSKMG zRCPNkp{kZzKnvIav|z0RtCdWq85reXV8`<<6ve$XEoRBO(J;D_d;44gi2*FX^C&Oa_WgipZe1?Z%!r0swdi?1{*{Q+yuS zGg}NmQU_eLJ`bim9owq~pkD`4Vh&U=8>RY&v_JwzuZ_=CK!+GgvHkjUJO-qsuj};uXvBtIC@&A43xy z2+kkk8~`{mJz4|6i0RWR0KS=?6Opmh$|5ojytJQRqJH_SIFMkP>>bx4WAKke3{S~U z!4>daRd<431tfZ7#jC!4&wvTNyz+R7IS9qdHm}t}Oi0lI-=QdT<Dxr8kKnJ=7d*gG;ChYvHc@*9h{!bX#JIEs zJSIx(OclT^u+NbzFs-V8R(3Lh5PyMFhU^dEgsT2kM1hDL0PX;DNpY;ZRsur20!Iwl z#iGFjsOo#*qH*bHX|GZcf`HEr+3ON|UKz44O6bW%R|%$F5Rn<+3h>7_$(oM1B@ol7l31_3Yx)h^w$R;fXT%9s;YiW zYn)jL#tk`;z|&yY7bxgNQ)oN`J^&j9oJhib zA{y(lbh{Ikc~8@=12#xZd;7kHk^@fNDa1wY5{E~6|k3{2w-vGB%^+t{_m1c#Y#6n3CSp*IyL~m7f+}Cg& z5Rtd^uVGqL)k8iy%1mpyEN`X0fOiJeLBLk}3$#MOurN&PEET=&h%VV~9oX-OKB}S^`B_KHM+d zzgCR({{qM64uz(>54eA@ud&ii>+UflPDYerC>??JJC&ZvbPjQnAql| zrP(iN@kx~aeDqI~^_>ShTh54{{c<2Yejm%)M}ns<4r@;1+J5cI^=JtNWP@y|RWA%5)A_)!U!=6OWa3ba%Tg}*|9=twC22}CI&L4ryr#PcK~ zB7W4Okq|`Wy`$K3-=4j@>+Q;A6+6i!v**m6`R?2~b9N)8WRfy7(d<}VUxmaYNZDwq7@Ohv$jXb zRmSY*D!OzHe&}@R6*@BPc@`II zucI|m>=6M@m&uEZYZ)M~l@Mo|1endGJzu*ydjRtK+3Y*l7{CyyfdJ#`e>DlG08@Y| zz!YFySsY@IC4GO*#*i1dgP(Z)r$PFD5BzJHcn|yn?X6@UmO5cF00000NkvXXu0mjf Dma6|+ literal 0 HcmV?d00001 diff --git a/resources/img/upload-64.png b/resources/img/upload-64.png new file mode 100644 index 0000000000000000000000000000000000000000..65629dfb4ae1193db0e6150f47a5887cc9c27d32 GIT binary patch literal 1006 zcmVXdZ>uB0FMYdPJn|V(xIw-89ic&!{^E=v;w|3 ze7jTA$%7+#fMr0hh_q+%44}o~`|P|jq@kz*#wczm&H`0sG%CA*QAgYvn*jEg*$f>6 z{uIQW2I^hcJKJFZ5xE0&I);XTcg(iK!@xCFeeMi28?P03$p16%!UPbx97JRra1L0P z=iW@v5*MOhRok3_B96m;67SUI7qF#_nNL&+KrQe+W~?coIW$&8IUT0BoWksQNQ_q8 z%PGpnfjuEH4>na^+b# zQu1k}aKE#*$TnqVE)0!tIl{7sKOtN3QTTe5 z)^Wj~2zdd#0M-Po;RUcJU=1&TH34gQ0jvpF!wX + + diff --git a/resources/svg/circle.svg b/resources/svg/circle.svg new file mode 100644 index 0000000..84f078d --- /dev/null +++ b/resources/svg/circle.svg @@ -0,0 +1,3 @@ + + + diff --git a/resources/svg/clipboarddata.svg b/resources/svg/clipboarddata.svg new file mode 100644 index 0000000..aebd8f5 --- /dev/null +++ b/resources/svg/clipboarddata.svg @@ -0,0 +1,3 @@ + + + diff --git a/resources/svg/download.svg b/resources/svg/download.svg new file mode 100644 index 0000000..595f3a5 --- /dev/null +++ b/resources/svg/download.svg @@ -0,0 +1,4 @@ + + + + diff --git a/resources/svg/eye.svg b/resources/svg/eye.svg new file mode 100644 index 0000000..2f1f62f --- /dev/null +++ b/resources/svg/eye.svg @@ -0,0 +1,4 @@ + + + + diff --git a/resources/svg/flag.svg b/resources/svg/flag.svg new file mode 100644 index 0000000..ef18dc3 --- /dev/null +++ b/resources/svg/flag.svg @@ -0,0 +1,3 @@ + + + diff --git a/resources/svg/gear.svg b/resources/svg/gear.svg new file mode 100644 index 0000000..3827d6a --- /dev/null +++ b/resources/svg/gear.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/resources/svg/reply.svg b/resources/svg/reply.svg new file mode 100644 index 0000000..fdf9414 --- /dev/null +++ b/resources/svg/reply.svg @@ -0,0 +1,4 @@ + + + + diff --git a/resources/svg/settings.svg b/resources/svg/settings.svg new file mode 100644 index 0000000..ce5be79 --- /dev/null +++ b/resources/svg/settings.svg @@ -0,0 +1,3 @@ + + + diff --git a/resources/svg/trash.svg b/resources/svg/trash.svg new file mode 100644 index 0000000..c236be7 --- /dev/null +++ b/resources/svg/trash.svg @@ -0,0 +1,3 @@ + + + diff --git a/resources/svg/upload.svg b/resources/svg/upload.svg new file mode 100644 index 0000000..09e7ce4 --- /dev/null +++ b/resources/svg/upload.svg @@ -0,0 +1,4 @@ + + + + diff --git a/resources/svg2img.ps1 b/resources/svg2img.ps1 new file mode 100644 index 0000000..c71f0ff --- /dev/null +++ b/resources/svg2img.ps1 @@ -0,0 +1,28 @@ +$svgDir = "./svg" +$outDir = "./img" +$sizes = @(16, 32, 64) + +# Ensure output directory exists +if (!(Test-Path -Path $outDir)) { + New-Item -ItemType Directory -Path $outDir | Out-Null +} + +# Check for Inkscape +if (-not (Get-Command "inkscape" -ErrorAction SilentlyContinue)) { + Write-Error "Inkscape CLI is not installed or not in PATH. Please install it from https://inkscape.org/" + exit 1 +} + +# Process SVGs +Get-ChildItem -Path $svgDir -Filter *.svg | ForEach-Object { + $svgPath = $_.FullName + $baseName = $_.BaseName + + foreach ($size in $sizes) { + $outFile = Join-Path $outDir "$baseName-$size.png" + Write-Host "Converting $($_.Name) to $outFile ($size x $size)..." + & inkscape "$svgPath" --export-type=png --export-filename="$outFile" --export-width=$size --export-height=$size + } +} + +Write-Host "Conversion complete." From 58a2891847eaaf7991ba72ef8cdd3aa3e2895a30 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 7 Jul 2025 23:55:38 -0500 Subject: [PATCH 053/124] Update ai-filter.sln --- ai-filter.sln | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ai-filter.sln b/ai-filter.sln index 7818aed..f41f23f 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -53,7 +53,29 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-1755-4A95-A11B-6C90C701C5BF}" ProjectSection(SolutionItems) = preProject + resources\img\average-16.png = resources\img\average-16.png + resources\img\average-32.png = resources\img\average-32.png + resources\img\average-64.png = resources\img\average-64.png + resources\img\circle-16.png = resources\img\circle-16.png + resources\img\circle-32.png = resources\img\circle-32.png + resources\img\circle-64.png = resources\img\circle-64.png + resources\img\clipboarddata-16.png = resources\img\clipboarddata-16.png + resources\img\clipboarddata-32.png = resources\img\clipboarddata-32.png + resources\img\clipboarddata-64.png = resources\img\clipboarddata-64.png + resources\img\download-16.png = resources\img\download-16.png + resources\img\download-32.png = resources\img\download-32.png + resources\img\download-64.png = resources\img\download-64.png + resources\img\eye-16.png = resources\img\eye-16.png + resources\img\eye-32.png = resources\img\eye-32.png + resources\img\eye-64.png = resources\img\eye-64.png + resources\img\flag-16.png = resources\img\flag-16.png + resources\img\flag-32.png = resources\img\flag-32.png + resources\img\flag-64.png = resources\img\flag-64.png + resources\img\full-logo-white.png = resources\img\full-logo-white.png resources\img\full-logo.png = resources\img\full-logo.png + resources\img\gear-16.png = resources\img\gear-16.png + resources\img\gear-32.png = resources\img\gear-32.png + resources\img\gear-64.png = resources\img\gear-64.png resources\img\logo.png = resources\img\logo.png resources\img\logo128.png = resources\img\logo128.png resources\img\logo16.png = resources\img\logo16.png @@ -61,6 +83,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "img", "img", "{F266602F-175 resources\img\logo48.png = resources\img\logo48.png resources\img\logo64.png = resources\img\logo64.png resources\img\logo96.png = resources\img\logo96.png + resources\img\reply-16.png = resources\img\reply-16.png + resources\img\reply-32.png = resources\img\reply-32.png + resources\img\reply-64.png = resources\img\reply-64.png + resources\img\settings-16.png = resources\img\settings-16.png + resources\img\settings-32.png = resources\img\settings-32.png + resources\img\settings-64.png = resources\img\settings-64.png + resources\img\trash-16.png = resources\img\trash-16.png + resources\img\trash-32.png = resources\img\trash-32.png + resources\img\trash-64.png = resources\img\trash-64.png + resources\img\upload-16.png = resources\img\upload-16.png + resources\img\upload-32.png = resources\img\upload-32.png + resources\img\upload-64.png = resources\img\upload-64.png EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "js", "js", "{21D2A42C-3F85-465C-9141-C106AFD92B68}" From 6c352e904e146fcf3f619d093f6905ac041aaf0e Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Tue, 8 Jul 2025 00:24:43 -0500 Subject: [PATCH 054/124] more images --- resources/img/check-16.png | Bin 0 -> 300 bytes resources/img/check-32.png | Bin 0 -> 450 bytes resources/img/check-64.png | Bin 0 -> 927 bytes resources/img/circledots-16.png | Bin 0 -> 394 bytes resources/img/circledots-32.png | Bin 0 -> 773 bytes resources/img/circledots-64.png | Bin 0 -> 1656 bytes resources/img/x-16.png | Bin 0 -> 227 bytes resources/img/x-32.png | Bin 0 -> 331 bytes resources/img/x-64.png | Bin 0 -> 655 bytes resources/svg/check.svg | 3 +++ resources/svg/circledots.svg | 4 ++++ resources/svg/x.svg | 3 +++ 12 files changed, 10 insertions(+) create mode 100644 resources/img/check-16.png create mode 100644 resources/img/check-32.png create mode 100644 resources/img/check-64.png create mode 100644 resources/img/circledots-16.png create mode 100644 resources/img/circledots-32.png create mode 100644 resources/img/circledots-64.png create mode 100644 resources/img/x-16.png create mode 100644 resources/img/x-32.png create mode 100644 resources/img/x-64.png create mode 100644 resources/svg/check.svg create mode 100644 resources/svg/circledots.svg create mode 100644 resources/svg/x.svg diff --git a/resources/img/check-16.png b/resources/img/check-16.png new file mode 100644 index 0000000000000000000000000000000000000000..2a829805b3a1cd134853c1ad26ec809c1048a6c7 GIT binary patch literal 300 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`oCO|{#X#yh2s3WX6WN&PEETXs4%(V~9oX)yuY7PJsff5At0u zbxbhPpS!G3_}zh9u1mz;Y-+f2v#B*Zfa6BDZxB~xW2!=e;`L{m&t86J=l*y|(nd_4)D+W|Fx{` zSH?7sJ;{yN;_}loIxIIa{S>XwkQdRfka^7AZ+Y#Y(A}Ww*4)xF29DDEU&x+)-f&-M s`=&$3Pd0cvRUeerTF&z5UwaF1JXNji0(O88K!ICe<=0Gw@DA{7@F!q1b*>`#DsX4;cVIhnwgUJPaBc7} zV1Md-$?z6%VQ>WwbN8f#w1K3cz*zu4joOnEa&7?wC;T{okAPNDf$$afBjG(eZw0P^ z#c1)Auw`Hzd_)I1@Z7GuKSE$FsL=%2&6tbox?8})H-JYAfBo5IPuB|x7-S2037)WH z_g!5tEMTy1aK;b7y0_~y37Etl@M*v|Z`WrQFo}Kuc6GfJ0h1UQ{L<6)X_r`V^b(N- sxL4JQYhzZPSD@_mfBuRafj=wo4O$HW=K^IzoB#j-07*qoM6N<$g5e&#HUIzs literal 0 HcmV?d00001 diff --git a/resources/img/check-64.png b/resources/img/check-64.png new file mode 100644 index 0000000000000000000000000000000000000000..175c4a276691fe7854413dbf6b0ec13e843c6a3d GIT binary patch literal 927 zcmV;Q17Q4#P)7@5gh`@7;4nRoTpH zvNm9?9Dua}Yvll}4OlA&U~Rx!IRI+|*2)1`8?aUmz}i645|M(442Vd3RUK5dVJ_f0 zpzitsbXMxqFe^YrhJdSU{01%Nr;8v`@Sw9dFkpl1%*jCc?3plK*)8+bS zdIg9`2Qbe1@Csb06dp1RKt$SrXTa`=<`GpLt7;(Yojc}kzXh0Nb;na~?3W&%cFdES`3v{dMmzu}S3IGx50UiM@B~1$y%iT^n9G2c9}F9`4|OmzZF5vKkCCld@`02U+WSptp*TX9vI>ve%k%`F4R z6A0e{eDr6MngBSn!ycnw<2<9JRO=H1 z;7t2U)JI@X4dW#GUY{la0q%N?zXNvG2{P7ueQE%l`DWmuC%{yt0KlN9Z;)D_H~<%7 zEAYw_;;2<0XzIpz`A3WjjScX?k_*7U#x`7w0|5O#Fr@$#y zT}jM93?+Ibrm9oGS&DCT<@5$PV;vp>oC&JxW8fmiB_5?&pjTBFlb$<{f*gvf>OJ5} zMAHIrLSuZu2&SMrEL>yt4RQfEV!i1F-2|f~B8Pwjz;jid3p!a0W#-dZC~fEuBWWlH zU~Rx!IRI+|*2)1`8?aUmz}kSdasbu_thMO@`~w5OS?n{{`V0U7002ovPDHLkV1mW! Bk2wGU literal 0 HcmV?d00001 diff --git a/resources/img/circledots-16.png b/resources/img/circledots-16.png new file mode 100644 index 0000000000000000000000000000000000000000..1443d226c15d61a0f9fa7c9a5e055a1d468258b0 GIT binary patch literal 394 zcmV;50d@X~P);L)DsdN}nXE&|t}_FxN( zl3MDS>i?^{|F04e#ziS*0iFDOj)<4B2P5KVpnU=$~Snxa$TX-VJFXYn@*a9Bhonm!KEi-^~TanH~s zohhoX7|OWaHqw=$NhXpc_z@RsBV}?nNj?m9UaxW+IL*qJ+E#82(=P7vk(yx|;1)Yt o!_;X1D@rNz*)lvu#M`)Y-~5wsLN!KCO#lD@07*qoM6N<$f}a1N?*IS* literal 0 HcmV?d00001 diff --git a/resources/img/circledots-32.png b/resources/img/circledots-32.png new file mode 100644 index 0000000000000000000000000000000000000000..e8dfa44211444bbbf9660f7e39b74fc6ac5ce183 GIT binary patch literal 773 zcmV+g1N!`lP)@(^qejM*MXzJF15ELx*HNDZSw-Cq#^(u0lp|F?|@TDLMrw(;Jvc{3=AhC z08~8iM?fJdxbkCwyFgo!*b<4*nMYx71NF4P0V0!%M8nSsfAcl<<@f#$9oTw%NwjKxS-05Hbv15V;( zj={+jnQL;TI|9H7;&YPf@`1hYRPn8*uN!H#={7!;*4BXu|IAYnP(;eWH1L}Kzh-}d zSHP4mwSL10Kw^>ZX$&|>_s{7j?&tJRDt~DS@?3cCbFv#5B5g)C845R3$b5Hv=w%93n+kgU;zcP|J;jRzwE z)WnMsh2Q~MB^m@oq8Cw&2$2XV@!*4(;r*VO{XA3;+f!X#(=$`uy%WBqveUIyU;V#- zRekkVBEroqEqMdTTXH2pDb=Br8k;MIw7Y;(>ULlrFb9|dbORHC@kY|Kz;WOZa2VJF zd?zBOQ^l7C0ZOU6fu+C_Uk+eYz7_& zS{op_Ezq$!&>2_WRp5ox2>=xEx&z2iP}`Gz^iQMgDZs};8FA&WN{xWP`1c%z{{lSZ z+i(kxhk^YL9P82|;0;Hi@4NbT3M7mI-gLBvm0k%zd-=G&_f%GfTX6L`e9?dx0)SaD zg`5W#=4E^c-y-yIYO9>yh6n)0VIS(Qjs)Wk5^R<9JJ8t(0c&DBeQwR(fweEDE4*D7 z0l;*iU{Sa_Z(|*p-?88=0Qc8Jz}FUK`?B=*br8@F9I)WtUK0VpY>Sc-8I1EDNSGTl zJX*2GDf!Hz;KRI)tAT`#7W{*@w2{@IQmPC1o61#L3794#M`!|3N_7JVsoaPu0{4o@ z@o-w(PE9}2CFx+Z7!M#KhtL}{VQE8;6cu74V2LH}0}r^e78~ZL&4`LF5Nu$cqLP^7 z=%!QjjWA77N{s^k0qzV%mw-D&Zwy*qM9Z^}61yCfo)Z+v-jf?JJzf?iR^IknW% z*Xjh#sqk;;^ZT+Cfw3ZT-9>RQO!Wf`gQ#7=hAP8aYUxAy+JQ?{4uc0pr02$$!PX#A zrl(e8d87uQ$70WxxKRM~%?wj))8l3l8}>ut50e1f)Nz3&u32QEC2mjn!}I~S0sm1+ z%n4wUG}(&Ku^gTtBIn8x6)iwS&Ym4@AnFqPctagx-HD(RED$m}sVrv;Yw~0eophw*aryRS)^da-yw!s~9`N zKNI z+fWB)+ZCPyjyr|b6t1=?6re9}V=Y+wW3ayE%vVKeCwfoNRC*0ql(*3#oQttvkT?u< z)J;I3ycc~V#8iG^B#;mof6;>P3XXdLi3tb_8Hj0K*MOxt8s8sN^8zeOq@#wyR>c$~ z=xeRFW@vaj_Uk+4@M>aRH5RHx6PQTPzB0q9P_@M-9W z6RxYWiEF-Y0s=4tF^;huO2DVU6F^%7MvnrX1h(M5EPW+upZ7vQ0InB(4=)Z>Ih+hy z#WM63RqRYM=-WL(+KoZpYWfe8z}v@#MjuRc0xN)*N&e8Ggx;t)#qhyP2YT&(56K^1 z6o9qB3K2BTEVA*IWp#92BpP18nHrIu| zkhBE-v?gh@Hl+mYMepQn5|NXB5DrTg4=2Z?ucgdEpUrmzccC9nsvw!5A5Q*@c4|j} z?}2Z{*u(RVFKq&xg$K`OMdVU$81f`wB*;zwdi;NiiK$R6J&G>?0000N&PEETsKC?3F~p+x>SS-e1_d6M`xTzT z268>KUoFeKz`$shs_bdLey*pzV_MtuT)nSF%$}Y>GNRsGlY&^!zTsA2GB227^So~k zx9EG3cL!MK7@L`7uo{27upo1%^(Es S(sBuCH-o3EpUXO@geCxx1x&*L literal 0 HcmV?d00001 diff --git a/resources/img/x-32.png b/resources/img/x-32.png new file mode 100644 index 0000000000000000000000000000000000000000..785d7befb5343bc41d835d7516845ad133dbdd3b GIT binary patch literal 331 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdz&H|6fVg?4`^&rfc{H*UVP*AeO zHKHUqKdq!Zu_%?Hyu4g5GcUV1Ik6yBFTW^#_B$IXprRX|E{-7)t#79}ay1(WxZd}8 zKRI{7ju)Y}NlS10nFnheJ(8kyNKyg zV3`*}dV7}cJ}6hQ@qpX`?)5?*jZqJ-GIh$ns|vouBrTM;W7RJEqyEhMvzhePEty}% z^5ss8Wk9!!@t)8}0u|jy5)0n@=&gAFeul7&N0X<@TW*y_sVyP>5y{)F-mk3S_|toU Zn}NyWzp!@RC!il0JYD@<);T3K0RZc!e%}B9 literal 0 HcmV?d00001 diff --git a/resources/img/x-64.png b/resources/img/x-64.png new file mode 100644 index 0000000000000000000000000000000000000000..37dce1ab667d77e328cbca34ef17ff6b429a6c16 GIT binary patch literal 655 zcmV;A0&x9_P)W z^#LFA(79r+0f&~&6j(av#u4)c$=6u=n!gpkq$B7LfhYdbPkttF{K$* + + diff --git a/resources/svg/circledots.svg b/resources/svg/circledots.svg new file mode 100644 index 0000000..6275a98 --- /dev/null +++ b/resources/svg/circledots.svg @@ -0,0 +1,4 @@ + + + + diff --git a/resources/svg/x.svg b/resources/svg/x.svg new file mode 100644 index 0000000..b209221 --- /dev/null +++ b/resources/svg/x.svg @@ -0,0 +1,3 @@ + + + From 9a4b1ce2390974bcebd891131b048a795ed5efb8 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Tue, 8 Jul 2025 00:40:40 -0500 Subject: [PATCH 055/124] Update icons and options UI --- AGENTS.md | 7 +++++ README.md | 2 ++ background.js | 61 +++++++++++++++++++++++++++++++++++++++----- manifest.json | 2 +- options/options.html | 50 ++++++++++++++++++++++++++++-------- 5 files changed, 103 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ab79c66..c51f40c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,3 +66,10 @@ text extracted from all text parts. properties. Any legacy `aiReasonCache` data is merged into `aiCache` the first time the add-on loads after an update. +### Icon Set Usage + +Toolbar and menu icons reside under `resources/img` and are provided in 16, 32 +and 64 pixel variants. When changing these icons, pass a dictionary mapping the +sizes to the paths in `browserAction.setIcon` or `messageDisplayAction.setIcon`. +Use `resources/svg/svg2img.ps1` to regenerate PNGs from the SVG sources. + diff --git a/README.md b/README.md index eff53b1..bdff14a 100644 --- a/README.md +++ b/README.md @@ -141,3 +141,5 @@ Sortana builds upon knowledge gained from open-source projects. In particular, how Thunderbird's WebExtension and experiment APIs can be extended. Their code provided invaluable guidance during development. +- Icons from [cc0-icons.jonh.eu](https://cc0-icons.jonh.eu/) are used under the CC0 license. + diff --git a/background.js b/background.js index 59bac43..2916c06 100644 --- a/background.js +++ b/background.js @@ -28,6 +28,25 @@ let altTextImages = false; let collapseWhitespace = false; let TurndownService = null; +const ICONS = { + logo: "resources/img/logo.png", + circledots: { + 16: "resources/img/circledots-16.png", + 32: "resources/img/circledots-32.png", + 64: "resources/img/circledots-64.png" + }, + circle: { + 16: "resources/img/circle-16.png", + 32: "resources/img/circle-32.png", + 64: "resources/img/circle-64.png" + }, + average: { + 16: "resources/img/average-16.png", + 32: "resources/img/average-32.png", + 64: "resources/img/average-64.png" + } +}; + function setIcon(path) { if (browser.browserAction) { browser.browserAction.setIcon({ path }); @@ -38,9 +57,9 @@ function setIcon(path) { } function updateActionIcon() { - let path = "resources/img/brain.png"; + let path = ICONS.logo; if (processing || queuedCount > 0) { - path = "resources/img/busy.png"; + path = ICONS.circledots; } setIcon(path); } @@ -201,7 +220,7 @@ async function applyAiRules(idsInput) { t.mean += delta / t.count; t.m2 += delta * (elapsed - t.mean); await storage.local.set({ classifyStats: t }); - showTransientIcon("resources/img/done.png"); + showTransientIcon(ICONS.circle); } catch (e) { processing = false; const elapsed = Date.now() - currentStart; @@ -215,7 +234,7 @@ async function applyAiRules(idsInput) { t.m2 += delta * (elapsed - t.mean); await storage.local.set({ classifyStats: t }); logger.aiLog("failed to apply AI rules", { level: 'error' }, e); - showTransientIcon("resources/img/error.png"); + showTransientIcon(ICONS.average); } }); } @@ -250,7 +269,7 @@ async function clearCacheForMessages(idsInput) { } if (keys.length) { await AiClassifier.removeCacheEntries(keys); - showTransientIcon("resources/img/done.png"); + showTransientIcon(ICONS.circle); } } @@ -340,33 +359,61 @@ async function clearCacheForMessages(idsInput) { id: "apply-ai-rules-list", title: "Apply AI Rules", contexts: ["message_list"], + icons: { + 16: "resources/img/eye-16.png", + 32: "resources/img/eye-32.png", + 64: "resources/img/eye-64.png" + } }); browser.menus.create({ id: "apply-ai-rules-display", title: "Apply AI Rules", contexts: ["message_display_action"], + icons: { + 16: "resources/img/eye-16.png", + 32: "resources/img/eye-32.png", + 64: "resources/img/eye-64.png" + } }); browser.menus.create({ id: "clear-ai-cache-list", title: "Clear AI Cache", contexts: ["message_list"], + icons: { + 16: "resources/img/trash-16.png", + 32: "resources/img/trash-32.png", + 64: "resources/img/trash-64.png" + } }); browser.menus.create({ id: "clear-ai-cache-display", title: "Clear AI Cache", contexts: ["message_display_action"], + icons: { + 16: "resources/img/trash-16.png", + 32: "resources/img/trash-32.png", + 64: "resources/img/trash-64.png" + } }); browser.menus.create({ id: "view-ai-reason-list", title: "View Reasoning", contexts: ["message_list"], - icons: { "16": "resources/img/brain.png" } + icons: { + 16: "resources/img/clipboarddata-16.png", + 32: "resources/img/clipboarddata-32.png", + 64: "resources/img/clipboarddata-64.png" + } }); browser.menus.create({ id: "view-ai-reason-display", title: "View Reasoning", contexts: ["message_display_action"], - icons: { "16": "resources/img/brain.png" } + icons: { + 16: "resources/img/clipboarddata-16.png", + 32: "resources/img/clipboarddata-32.png", + 64: "resources/img/clipboarddata-64.png" + } }); browser.menus.onClicked.addListener(async (info, tab) => { diff --git a/manifest.json b/manifest.json index 36b94b0..d44fd1d 100644 --- a/manifest.json +++ b/manifest.json @@ -22,7 +22,7 @@ "default_icon": "resources/img/logo32.png" }, "message_display_action": { - "default_icon": "resources/img/brain.png", + "default_icon": "resources/img/logo.png", "default_title": "Details", "default_label": "Details", "default_popup": "details.html" diff --git a/options/options.html b/options/options.html index 186e2b6..0fc5cae 100644 --- a/options/options.html +++ b/options/options.html @@ -44,18 +44,25 @@
- +
+

+ + Settings +

@@ -88,8 +95,14 @@
- - + +
+ diff --git a/options/options.js b/options/options.js index ebd7497..3a09d37 100644 --- a/options/options.js +++ b/options/options.js @@ -21,7 +21,9 @@ document.addEventListener('DOMContentLoaded', async () => { 'aiCache', 'theme', 'showDebugTab', - 'lastPayload' + 'lastPayload', + 'lastFullText', + 'lastPromptText' ]); const tabButtons = document.querySelectorAll('#main-tabs li'); const tabs = document.querySelectorAll('.tab-content'); @@ -67,9 +69,17 @@ document.addEventListener('DOMContentLoaded', async () => { await applyTheme(themeSelect.value); const payloadDisplay = document.getElementById('payload-display'); + const diffDisplay = document.getElementById('diff-display'); if (defaults.lastPayload) { payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2); } + if (defaults.lastFullText && defaults.lastPromptText && diff_match_patch) { + const dmp = new diff_match_patch(); + dmp.Diff_EditCost = 4; + const diffs = dmp.diff_main(defaults.lastFullText, defaults.lastPromptText); + dmp.diff_cleanupEfficiency(diffs); + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + } themeSelect.addEventListener('change', async () => { markDirty(); await applyTheme(themeSelect.value); From 841a697c69a9204bcaf7c22c30be8fd5259837f5 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 19:46:44 -0500 Subject: [PATCH 100/124] Update debug tab with live refresh --- README.md | 2 +- options/options.js | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 64fbf26..82649d0 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ message meets a specified criterion. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. -- **Debug tab** – view the last request payload sent to the AI service. +- **Debug tab** – view the last request payload and message diff with live updates. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/options/options.js b/options/options.js index 3a09d37..3402dcf 100644 --- a/options/options.js +++ b/options/options.js @@ -70,13 +70,18 @@ document.addEventListener('DOMContentLoaded', async () => { await applyTheme(themeSelect.value); const payloadDisplay = document.getElementById('payload-display'); const diffDisplay = document.getElementById('diff-display'); - if (defaults.lastPayload) { - payloadDisplay.textContent = JSON.stringify(defaults.lastPayload, null, 2); + + let lastFullText = defaults.lastFullText || ''; + let lastPromptText = defaults.lastPromptText || ''; + let lastPayload = defaults.lastPayload ? JSON.stringify(defaults.lastPayload, null, 2) : ''; + + if (lastPayload) { + payloadDisplay.textContent = lastPayload; } - if (defaults.lastFullText && defaults.lastPromptText && diff_match_patch) { + if (lastFullText && lastPromptText && diff_match_patch) { const dmp = new diff_match_patch(); dmp.Diff_EditCost = 4; - const diffs = dmp.diff_main(defaults.lastFullText, defaults.lastPromptText); + const diffs = dmp.diff_main(lastFullText, lastPromptText); dmp.diff_cleanupEfficiency(diffs); diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); } @@ -729,6 +734,30 @@ document.addEventListener('DOMContentLoaded', async () => { } catch { cacheCountEl.textContent = '?'; } + + try { + if (debugTabToggle.checked) { + const latest = await storage.local.get(['lastPayload', 'lastFullText', 'lastPromptText']); + const payloadStr = latest.lastPayload ? JSON.stringify(latest.lastPayload, null, 2) : ''; + if (payloadStr !== lastPayload) { + lastPayload = payloadStr; + payloadDisplay.textContent = payloadStr; + } + if (latest.lastFullText !== lastFullText || latest.lastPromptText !== lastPromptText) { + lastFullText = latest.lastFullText || ''; + lastPromptText = latest.lastPromptText || ''; + if (lastFullText && lastPromptText && diff_match_patch) { + const dmp = new diff_match_patch(); + dmp.Diff_EditCost = 4; + const diffs = dmp.diff_main(lastFullText, lastPromptText); + dmp.diff_cleanupEfficiency(diffs); + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + } else { + diffDisplay.innerHTML = ''; + } + } + } + } catch {} } refreshMaintenance(); From bcac4ad01709c85002e2f8123406bacffcf1d4cd Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 21:58:37 -0500 Subject: [PATCH 101/124] Improve debug tab layout --- options/options.html | 15 +++++++++------ options/options.js | 22 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/options/options.html b/options/options.html index 3618a20..ddc5ee0 100644 --- a/options/options.html +++ b/options/options.html @@ -154,6 +154,11 @@ Aggressive token reduction
+
+ +
@@ -220,11 +225,6 @@
-
- -
@@ -290,7 +290,10 @@ Debug

-                
+ diff --git a/options/options.js b/options/options.js index 3402dcf..d881d6a 100644 --- a/options/options.js +++ b/options/options.js @@ -70,6 +70,7 @@ document.addEventListener('DOMContentLoaded', async () => { await applyTheme(themeSelect.value); const payloadDisplay = document.getElementById('payload-display'); const diffDisplay = document.getElementById('diff-display'); + const diffContainer = document.getElementById('diff-container'); let lastFullText = defaults.lastFullText || ''; let lastPromptText = defaults.lastPromptText || ''; @@ -83,7 +84,16 @@ document.addEventListener('DOMContentLoaded', async () => { dmp.Diff_EditCost = 4; const diffs = dmp.diff_main(lastFullText, lastPromptText); dmp.diff_cleanupEfficiency(diffs); - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + const hasDiff = diffs.some(d => d[0] !== 0); + if (hasDiff) { + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + diffContainer.classList.remove('is-hidden'); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + } + } else { + diffContainer.classList.add('is-hidden'); } themeSelect.addEventListener('change', async () => { markDirty(); @@ -751,9 +761,17 @@ document.addEventListener('DOMContentLoaded', async () => { dmp.Diff_EditCost = 4; const diffs = dmp.diff_main(lastFullText, lastPromptText); dmp.diff_cleanupEfficiency(diffs); - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + const hasDiff = diffs.some(d => d[0] !== 0); + if (hasDiff) { + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + diffContainer.classList.remove('is-hidden'); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + } } else { diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); } } } From 9cad2674e3841fbc40d237f7233fbd6784f61760 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 22:44:04 -0500 Subject: [PATCH 102/124] Capture raw message text for debug --- README.md | 2 +- background.js | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 82649d0..57a0285 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ message meets a specified criterion. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. -- **Debug tab** – view the last request payload and message diff with live updates. +- **Debug tab** – view the last request payload and a diff between the unaltered message text and the final prompt. - **Light/Dark themes** – automatically match Thunderbird's appearance with optional manual override. - **Automatic rules** – create rules that tag, move, copy, forward, reply, delete, archive, mark read/unread or flag/unflag messages based on AI classification. Rules can optionally apply only to unread messages and can ignore messages outside a chosen age range. - **Rule ordering** – drag rules to prioritize them and optionally stop processing after a match. diff --git a/background.js b/background.js index 58a5de5..fc585ff 100644 --- a/background.js +++ b/background.js @@ -210,17 +210,38 @@ function collectText(part, bodyParts, attachments) { } } -function buildEmailText(full) { +function collectRawText(part, bodyParts, attachments) { + if (part.parts && part.parts.length) { + for (const p of part.parts) collectRawText(p, bodyParts, attachments); + return; + } + const ct = (part.contentType || "text/plain").toLowerCase(); + const cd = (part.headers?.["content-disposition"]?.[0] || "").toLowerCase(); + const body = String(part.body || ""); + if (cd.includes("attachment") || !ct.startsWith("text/")) { + const nameMatch = /filename\s*=\s*"?([^";]+)/i.exec(cd) || /name\s*=\s*"?([^";]+)/i.exec(part.headers?.["content-type"]?.[0] || ""); + const name = nameMatch ? nameMatch[1] : ""; + attachments.push(`${name} (${ct}, ${part.size || byteSize(body)} bytes)`); + } else if (ct.startsWith("text/html")) { + const doc = new DOMParser().parseFromString(body, 'text/html'); + bodyParts.push(doc.body.textContent || ""); + } else { + bodyParts.push(body); + } +} + +function buildEmailText(full, applyTransforms = true) { const bodyParts = []; const attachments = []; - collectText(full, bodyParts, attachments); + const collect = applyTransforms ? collectText : collectRawText; + collect(full, bodyParts, attachments); const headers = Object.entries(full.headers || {}) .map(([k, v]) => `${k}: ${v.join(' ')}`) .join('\n'); const attachInfo = `Attachments: ${attachments.length}` + (attachments.length ? "\n" + attachments.map(a => ` - ${a}`).join('\n') : ""); let combined = `${headers}\n${attachInfo}\n\n${bodyParts.join('\n')}`.trim(); - if (tokenReduction) { + if (applyTransforms && tokenReduction) { const seen = new Set(); combined = combined.split('\n').filter(l => { if (seen.has(l)) return false; @@ -228,7 +249,7 @@ function buildEmailText(full) { return true; }).join('\n'); } - return sanitizeString(combined); + return applyTransforms ? sanitizeString(combined) : combined; } function updateTimingStats(elapsed) { @@ -262,8 +283,8 @@ async function processMessage(id) { updateActionIcon(); try { const full = await messenger.messages.getFull(id); + const originalText = buildEmailText(full, false); let text = buildEmailText(full); - const originalText = text; if (tokenReduction && maxTokens > 0) { const limit = Math.floor(maxTokens * 0.9); if (text.length > limit) { From c622c07c66a6936c4c8d3bb039b1f0bba4692384 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Mon, 18 Aug 2025 19:39:35 -0500 Subject: [PATCH 103/124] Supporting v140+ After testing in Betterbird v140, updating manifest. --- manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index a3c9f7c..e7cb9d8 100644 --- a/manifest.json +++ b/manifest.json @@ -1,13 +1,13 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.1.2", + "version": "2.2.0", "default_locale": "en-US", "applications": { "gecko": { "id": "ai-filter@jordanwages", "strict_min_version": "128.0", - "strict_max_version": "139.*" + "strict_max_version": "140.*" } }, "icons": { From 0f2f148b71913b89489cf72ab6bc0efc263c1064 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 20:45:31 -0600 Subject: [PATCH 104/124] Normalize completions endpoint base --- AGENTS.md | 5 ++++- README.md | 11 ++++++----- modules/AiClassifier.js | 41 +++++++++++++++++++++++++++++++++++++---- options/options.html | 1 + options/options.js | 18 ++++++++++++++++-- 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 401f962..e2a1696 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,10 @@ This file provides guidelines for codex agents contributing to the Sortana proje There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. Do **not** run the `ps1` build script or the SVG processing script. +## Endpoint Notes + +Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. + ## Documentation Additional documentation exists outside this repository. @@ -73,4 +77,3 @@ Toolbar and menu icons reside under `resources/img` and are provided in 16, 32 and 64 pixel variants. When changing these icons, pass a dictionary mapping the sizes to the paths in `browserAction.setIcon` or `messageDisplayAction.setIcon`. Use `resources/svg2img.ps1` to regenerate PNGs from the SVG sources. - diff --git a/README.md b/README.md index 57a0285..2f8f204 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,13 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filter rule. It allows you to classify email messages by sending their contents to a configurable -HTTP endpoint. The endpoint should respond with JSON indicating whether the -message meets a specified criterion. +HTTP endpoint. Sortana uses the `/v1/completions` API; the options page stores a base +URL and appends `/v1/completions` when sending requests. The endpoint should respond +with JSON indicating whether the message meets a specified criterion. ## Features -- **Configurable endpoint** – set the classification service URL on the options page. +- **Configurable endpoint** – set the classification service base URL on the options page. - **Prompt templates** – choose between several model formats or provide your own custom template. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. @@ -72,7 +73,8 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e ## Usage -1. Open the add-on's options and set the URL of your classification service. +1. Open the add-on's options and set the base URL of your classification service + (Sortana will append `/v1/completions`). 2. Use the **Classification Rules** section to add a criterion and optional actions such as tagging, moving, copying, forwarding, replying, deleting or archiving a message when it matches. Drag rules to @@ -158,4 +160,3 @@ how Thunderbird's WebExtension and experiment APIs can be extended. Their code provided invaluable guidance during development. - Icons from [cc0-icons.jonh.eu](https://cc0-icons.jonh.eu/) are used under the CC0 license. - diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 8313654..f5a3bff 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -15,6 +15,8 @@ try { Services = undefined; } +const COMPLETIONS_PATH = "/v1/completions"; + const SYSTEM_PREFIX = `You are an email-classification assistant. Read the email below and the classification criterion provided by the user. `; @@ -28,7 +30,8 @@ Return ONLY a JSON object on a single line of the form: Do not add any other keys, text, or formatting.`; -let gEndpoint = "http://127.0.0.1:5000/v1/classify"; +let gEndpointBase = "http://127.0.0.1:5000"; +let gEndpoint = buildEndpointUrl(gEndpointBase); let gTemplateName = "openai"; let gCustomTemplate = ""; let gCustomSystemPrompt = DEFAULT_CUSTOM_SYSTEM_PROMPT; @@ -39,6 +42,28 @@ let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); let gCache = new Map(); let gCacheLoaded = false; +function normalizeEndpointBase(endpoint) { + if (typeof endpoint !== "string") { + return ""; + } + let base = endpoint.trim(); + if (!base) { + return ""; + } + base = base.replace(/\/v1\/completions\/?$/i, ""); + return base; +} + +function buildEndpointUrl(endpointBase) { + const base = normalizeEndpointBase(endpointBase); + if (!base) { + return ""; + } + const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; + const needsSlash = withScheme.endsWith("/"); + return `${withScheme}${needsSlash ? "" : "/"}v1/completions`; +} + function sha256HexSync(str) { try { const hasher = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); @@ -158,8 +183,12 @@ function loadTemplateSync(name) { } async function setConfig(config = {}) { - if (config.endpoint) { - gEndpoint = config.endpoint; + if (typeof config.endpoint === "string") { + const base = normalizeEndpointBase(config.endpoint); + if (base) { + gEndpointBase = base; + } + gEndpoint = buildEndpointUrl(gEndpointBase); } if (config.templateName) { gTemplateName = config.templateName; @@ -187,6 +216,10 @@ async function setConfig(config = {}) { } else { gTemplateText = await loadTemplate(gTemplateName); } + if (!gEndpoint) { + gEndpoint = buildEndpointUrl(gEndpointBase); + } + aiLog(`[AiClassifier] Endpoint base set to ${gEndpointBase}`, {debug: true}); aiLog(`[AiClassifier] Endpoint set to ${gEndpoint}`, {debug: true}); aiLog(`[AiClassifier] Template set to ${gTemplateName}`, {debug: true}); } @@ -344,4 +377,4 @@ async function init() { await loadCache(); } -export { classifyText, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, getCacheSize, init }; +export { buildEndpointUrl, normalizeEndpointBase, classifyText, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, getCacheSize, init }; diff --git a/options/options.html b/options/options.html index ddc5ee0..59ebc83 100644 --- a/options/options.html +++ b/options/options.html @@ -73,6 +73,7 @@
+

diff --git a/options/options.js b/options/options.js index d881d6a..30fa5f3 100644 --- a/options/options.js +++ b/options/options.js @@ -99,7 +99,21 @@ document.addEventListener('DOMContentLoaded', async () => { markDirty(); await applyTheme(themeSelect.value); }); - document.getElementById('endpoint').value = defaults.endpoint || 'http://127.0.0.1:5000/v1/completions'; + const endpointInput = document.getElementById('endpoint'); + const endpointPreview = document.getElementById('endpoint-preview'); + const fallbackEndpoint = 'http://127.0.0.1:5000'; + const storedEndpoint = defaults.endpoint || fallbackEndpoint; + const endpointBase = AiClassifier.normalizeEndpointBase(storedEndpoint) || storedEndpoint; + endpointInput.value = endpointBase; + + function updateEndpointPreview() { + const resolved = AiClassifier.buildEndpointUrl(endpointInput.value); + endpointPreview.textContent = resolved + ? `Resolved endpoint: ${resolved}` + : 'Resolved endpoint: (invalid)'; + } + endpointInput.addEventListener('input', updateEndpointPreview); + updateEndpointPreview(); const templates = { openai: browser.i18n.getMessage('template.openai'), @@ -806,7 +820,7 @@ document.addEventListener('DOMContentLoaded', async () => { initialized = true; document.getElementById('save').addEventListener('click', async () => { - const endpoint = document.getElementById('endpoint').value; + const endpoint = endpointInput.value.trim(); const templateName = templateSelect.value; const customTemplateText = customTemplate.value; const customSystemPrompt = systemBox.value; From 2178de9a90ba4ea8c846d839f138de183c0186a2 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 21:07:55 -0600 Subject: [PATCH 105/124] Add bash build script --- AGENTS.md | 1 + README.md | 7 +++-- build-xpi.sh | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100755 build-xpi.sh diff --git a/AGENTS.md b/AGENTS.md index e2a1696..23080ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ This file provides guidelines for codex agents contributing to the Sortana proje - `resources/`: Images and other static files. - `prompt_templates/`: Prompt template files for the AI service. - `build-xpi.ps1`: PowerShell script to package the extension. +- `build-xpi.sh`: Bash script to package the extension. ## Coding Style diff --git a/README.md b/README.md index 2f8f204..a58a799 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ with JSON indicating whether the message meets a specified criterion. - **View reasoning** – inspect why rules matched via the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. -- **Packaging script** – `build-xpi.ps1` builds an XPI ready for installation. +- **Packaging scripts** – `build-xpi.ps1` (PowerShell) or `build-xpi.sh` (bash) build an XPI ready for installation. - **Maintenance tab** – view rule counts, cache entries and clear cached results from the options page. ### Cache Storage @@ -65,8 +65,9 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e 1. Ensure PowerShell is available (for Windows) or adapt the script for other environments. 2. The Bulma stylesheet (v1.0.3) is already included as `options/bulma.css`. -3. Run `powershell ./build-xpi.ps1` from the repository root. The script reads - the version from `manifest.json` and creates an XPI in the `release` folder. +3. Run `powershell ./build-xpi.ps1` or `./build-xpi.sh` from the repository root. + The script reads the version from `manifest.json` and creates an XPI in the + `release` folder. 4. Install the generated XPI in Thunderbird via the Add-ons Manager. During development you can also load the directory as a temporary add-on. 5. To regenerate PNG icons from the SVG sources, run `resources/svg2img.ps1`. diff --git a/build-xpi.sh b/build-xpi.sh new file mode 100755 index 0000000..20c6e15 --- /dev/null +++ b/build-xpi.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +release_dir="$script_dir/release" +manifest="$script_dir/manifest.json" + +if [[ ! -f "$manifest" ]]; then + echo "manifest.json not found at $manifest" >&2 + exit 1 +fi + +if ! command -v zip >/dev/null 2>&1; then + echo "zip is required to build the XPI." >&2 + exit 1 +fi + +if command -v jq >/dev/null 2>&1; then + version="$(jq -r '.version // empty' "$manifest")" +else + if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to read manifest.json without jq." >&2 + exit 1 + fi + version="$(python3 - <<'PY' +import json +import sys +with open(sys.argv[1], 'r', encoding='utf-8') as f: + data = json.load(f) +print(data.get('version', '') or '') +PY +"$manifest")" +fi + +if [[ -z "$version" ]]; then + echo "No version found in manifest.json" >&2 + exit 1 +fi + +mkdir -p "$release_dir" + +xpi_name="sortana-$version.xpi" +zip_path="$release_dir/ai-filter-$version.zip" +xpi_path="$release_dir/$xpi_name" + +rm -f "$zip_path" "$xpi_path" + +mapfile -d '' files < <( + find "$script_dir" -type f \ + ! -name '*.sln' \ + ! -name '*.ps1' \ + ! -name '*.sh' \ + ! -path "$release_dir/*" \ + ! -path "$script_dir/.vs/*" \ + ! -path "$script_dir/.git/*" \ + -printf '%P\0' +) + +if [[ ${#files[@]} -eq 0 ]]; then + echo "No files found to package." >&2 + exit 0 +fi + +for rel in "${files[@]}"; do + full="$script_dir/$rel" + size=$(stat -c '%s' "$full") + echo "Zipping: $rel <- $full ($size bytes)" +done + +( + cd "$script_dir" + printf '%s\n' "${files[@]}" | zip -q -9 -@ "$zip_path" +) + +mv -f "$zip_path" "$xpi_path" + +echo "Built XPI at: $xpi_path" From af6702bceb57fa65fda3d47c218d42b40cc83bfe Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 21:42:57 -0600 Subject: [PATCH 106/124] Add prompt reduction badge to debug diff --- options/options.html | 5 ++- options/options.js | 88 +++++++++++++++++++++++++++----------------- 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/options/options.html b/options/options.html index 59ebc83..b118ee0 100644 --- a/options/options.html +++ b/options/options.html @@ -292,7 +292,10 @@

                 
             
diff --git a/options/options.js b/options/options.js index 30fa5f3..046c674 100644 --- a/options/options.js +++ b/options/options.js @@ -71,6 +71,7 @@ document.addEventListener('DOMContentLoaded', async () => { const payloadDisplay = document.getElementById('payload-display'); const diffDisplay = document.getElementById('diff-display'); const diffContainer = document.getElementById('diff-container'); + const promptReductionLabel = document.getElementById('prompt-reduction'); let lastFullText = defaults.lastFullText || ''; let lastPromptText = defaults.lastPromptText || ''; @@ -79,22 +80,6 @@ document.addEventListener('DOMContentLoaded', async () => { if (lastPayload) { payloadDisplay.textContent = lastPayload; } - if (lastFullText && lastPromptText && diff_match_patch) { - const dmp = new diff_match_patch(); - dmp.Diff_EditCost = 4; - const diffs = dmp.diff_main(lastFullText, lastPromptText); - dmp.diff_cleanupEfficiency(diffs); - const hasDiff = diffs.some(d => d[0] !== 0); - if (hasDiff) { - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); - diffContainer.classList.remove('is-hidden'); - } else { - diffDisplay.innerHTML = ''; - diffContainer.classList.add('is-hidden'); - } - } else { - diffContainer.classList.add('is-hidden'); - } themeSelect.addEventListener('change', async () => { markDirty(); await applyTheme(themeSelect.value); @@ -164,6 +149,51 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReductionToggle = document.getElementById('token-reduction'); tokenReductionToggle.checked = defaults.tokenReduction === true; + function tokenSavingEnabled() { + return htmlToggle.checked + || stripUrlToggle.checked + || altTextToggle.checked + || collapseWhitespaceToggle.checked + || tokenReductionToggle.checked; + } + + function updatePromptReductionLabel(hasDiff) { + if (!promptReductionLabel) return; + if (!hasDiff || !tokenSavingEnabled() || !lastFullText || !lastPromptText) { + promptReductionLabel.classList.add('is-hidden'); + return; + } + const baseLength = lastFullText.length; + const promptLength = lastPromptText.length; + const percentSaved = baseLength > 0 + ? Math.max(0, Math.round((1 - (promptLength / baseLength)) * 100)) + : 0; + promptReductionLabel.textContent = `Prompt Token Reduction: ${percentSaved}%`; + promptReductionLabel.classList.remove('is-hidden'); + } + + function updateDiffDisplay() { + if (lastFullText && lastPromptText && diff_match_patch) { + const dmp = new diff_match_patch(); + dmp.Diff_EditCost = 4; + const diffs = dmp.diff_main(lastFullText, lastPromptText); + dmp.diff_cleanupEfficiency(diffs); + const hasDiff = diffs.some(d => d[0] !== 0); + if (hasDiff) { + diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); + diffContainer.classList.remove('is-hidden'); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + } + updatePromptReductionLabel(hasDiff); + } else { + diffDisplay.innerHTML = ''; + diffContainer.classList.add('is-hidden'); + updatePromptReductionLabel(false); + } + } + const debugTabToggle = document.getElementById('show-debug-tab'); const debugTabBtn = document.getElementById('debug-tab-button'); function updateDebugTab() { @@ -174,6 +204,14 @@ document.addEventListener('DOMContentLoaded', async () => { debugTabToggle.addEventListener('change', () => { updateDebugTab(); markDirty(); }); updateDebugTab(); + updateDiffDisplay(); + + [htmlToggle, stripUrlToggle, altTextToggle, collapseWhitespaceToggle, tokenReductionToggle].forEach(toggle => { + toggle.addEventListener('change', () => { + updatePromptReductionLabel(!diffContainer.classList.contains('is-hidden')); + }); + }); + const aiParams = Object.assign({}, DEFAULT_AI_PARAMS, defaults.aiParams || {}); for (const [key, val] of Object.entries(aiParams)) { @@ -770,23 +808,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (latest.lastFullText !== lastFullText || latest.lastPromptText !== lastPromptText) { lastFullText = latest.lastFullText || ''; lastPromptText = latest.lastPromptText || ''; - if (lastFullText && lastPromptText && diff_match_patch) { - const dmp = new diff_match_patch(); - dmp.Diff_EditCost = 4; - const diffs = dmp.diff_main(lastFullText, lastPromptText); - dmp.diff_cleanupEfficiency(diffs); - const hasDiff = diffs.some(d => d[0] !== 0); - if (hasDiff) { - diffDisplay.innerHTML = dmp.diff_prettyHtml(diffs); - diffContainer.classList.remove('is-hidden'); - } else { - diffDisplay.innerHTML = ''; - diffContainer.classList.add('is-hidden'); - } - } else { - diffDisplay.innerHTML = ''; - diffContainer.classList.add('is-hidden'); - } + updateDiffDisplay(); } } } catch {} From 9269225a0c5da9e13a024677be8375694ac7d93a Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Tue, 6 Jan 2026 22:01:20 -0600 Subject: [PATCH 107/124] Add session error log and transient error icon --- README.md | 7 +++-- background.js | 46 +++++++++++++++++++++-------- options/options.html | 27 +++++++++++++++++ options/options.js | 70 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a58a799..87a3565 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,9 @@ with JSON indicating whether the message meets a specified criterion. - **Rule enable/disable** – temporarily turn a rule off without removing it. - **Account & folder filters** – limit rules to specific accounts or folders. - **Context menu** – apply AI rules from the message list or the message display action button. -- **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red until you dismiss the notification. -- **Error notification** – failed classification displays a notification with a button to clear the error and reset the icon. +- **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red briefly before returning to normal. +- **Error notification** – failed classification displays a notification in Thunderbird. +- **Session error log** – the Errors tab (visible only when errors occur) shows errors recorded since the last add-on start. - **View reasoning** – inspect why rules matched via the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. @@ -88,7 +89,7 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e open a compose window using the account that received the message. 3. Save your settings. New mail will be evaluated automatically using the configured rules. -4. If the toolbar icon shows a red X, click the notification's **Dismiss** button to clear the error. +4. If the toolbar icon shows a red X, it will clear after a few seconds. Open the Errors tab in Options to review the latest failures. ### Example Filters diff --git a/background.js b/background.js index fc585ff..e16d5a3 100644 --- a/background.js +++ b/background.js @@ -19,6 +19,7 @@ let queue = Promise.resolve(); let queuedCount = 0; let processing = false; let iconTimer = null; +let errorTimer = null; let timingStats = { count: 0, mean: 0, m2: 0, total: 0, last: -1 }; let currentStart = 0; let logGetTiming = true; @@ -33,8 +34,11 @@ let userTheme = 'auto'; let currentTheme = 'light'; let detectSystemTheme; let errorPending = false; +let errorLog = []; let showDebugTab = false; const ERROR_NOTIFICATION_ID = 'sortana-error'; +const ERROR_ICON_TIMEOUT = 4500; +const MAX_ERROR_LOG = 50; function normalizeRules(rules) { return Array.isArray(rules) ? rules.map(r => { @@ -108,11 +112,33 @@ function showTransientIcon(factory, delay = 1500) { async function clearError() { errorPending = false; - await storage.local.set({ errorPending: false }); + clearTimeout(errorTimer); await browser.notifications.clear(ERROR_NOTIFICATION_ID); updateActionIcon(); } +function recordError(context, err) { + const message = err instanceof Error ? err.message : String(err || 'Unknown error'); + const detail = err instanceof Error ? err.stack : ''; + errorLog.unshift({ + time: Date.now(), + context, + message, + detail + }); + if (errorLog.length > MAX_ERROR_LOG) { + errorLog.length = MAX_ERROR_LOG; + } + errorPending = true; + updateActionIcon(); + clearTimeout(errorTimer); + errorTimer = setTimeout(() => { + errorPending = false; + updateActionIcon(); + }, ERROR_ICON_TIMEOUT); + browser.runtime.sendMessage({ type: 'sortana:errorLogUpdated', count: errorLog.length }).catch(() => {}); +} + function refreshMenuIcons() { browser.menus.update('apply-ai-rules-list', { icons: iconPaths('eye') }); browser.menus.update('apply-ai-rules-display', { icons: iconPaths('eye') }); @@ -382,16 +408,14 @@ async function processMessage(id) { const elapsed = Date.now() - currentStart; currentStart = 0; updateTimingStats(elapsed); - await storage.local.set({ classifyStats: timingStats, errorPending: true }); - errorPending = true; + await storage.local.set({ classifyStats: timingStats }); logger.aiLog("failed to apply AI rules", { level: 'error' }, e); - setIcon(ICONS.error()); + recordError("Failed to apply AI rules", e); browser.notifications.create(ERROR_NOTIFICATION_ID, { type: 'basic', iconUrl: browser.runtime.getURL('resources/img/logo.png'), title: 'Sortana Error', - message: 'Failed to apply AI rules', - buttons: [{ title: 'Dismiss' }] + message: 'Failed to apply AI rules' }); } } @@ -451,7 +475,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "errorPending", "showDebugTab"]); + const store = await storage.local.get(["endpoint", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -465,7 +489,6 @@ async function clearCacheForMessages(idsInput) { if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') { maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; } - errorPending = store.errorPending === true; showDebugTab = store.showDebugTab === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -524,10 +547,6 @@ async function clearCacheForMessages(idsInput) { if (changes.showDebugTab) { showDebugTab = changes.showDebugTab.newValue === true; } - if (changes.errorPending) { - errorPending = changes.errorPending.newValue === true; - updateActionIcon(); - } if (changes.theme) { userTheme = changes.theme.newValue || 'auto'; currentTheme = userTheme === 'auto' ? await detectSystemTheme() : userTheme; @@ -720,6 +739,8 @@ async function clearCacheForMessages(idsInput) { } } else if (msg?.type === "sortana:getQueueCount") { return { count: queuedCount + (processing ? 1 : 0) }; + } else if (msg?.type === "sortana:getErrorLog") { + return { errors: errorLog.slice() }; } else if (msg?.type === "sortana:getTiming") { const t = timingStats; const std = t.count > 1 ? Math.sqrt(t.m2 / (t.count - 1)) : 0; @@ -751,6 +772,7 @@ async function clearCacheForMessages(idsInput) { // Catch any unhandled rejections window.addEventListener("unhandledrejection", ev => { logger.aiLog("Unhandled promise rejection", { level: 'error' }, ev.reason); + recordError("Unhandled promise rejection", ev.reason); }); browser.notifications.onClicked.addListener(id => { diff --git a/options/options.html b/options/options.html index b118ee0..fd8700c 100644 --- a/options/options.html +++ b/options/options.html @@ -51,6 +51,7 @@
  • Settings
  • Rules
  • Maintenance
  • + @@ -285,6 +286,32 @@ + + +
    + +
    +
    +
    + +
    +
    +
    + +
    +
    +

    +
    +
    diff --git a/options/options.js b/options/options.js index 860c944..5e46857 100644 --- a/options/options.js +++ b/options/options.js @@ -10,6 +10,7 @@ document.addEventListener('DOMContentLoaded', async () => { 'templateName', 'customTemplate', 'customSystemPrompt', + 'model', 'aiParams', 'debugLogging', 'htmlToMarkdown', @@ -100,6 +101,88 @@ document.addEventListener('DOMContentLoaded', async () => { endpointInput.addEventListener('input', updateEndpointPreview); updateEndpointPreview(); + const modelSelect = document.getElementById('model-select'); + const refreshModelsBtn = document.getElementById('refresh-models'); + const modelHelp = document.getElementById('model-help'); + const storedModel = typeof defaults.model === 'string' ? defaults.model : ''; + + function setModelHelp(message = '', isError = false) { + if (!modelHelp) return; + modelHelp.textContent = message; + modelHelp.classList.toggle('is-danger', isError); + } + + function populateModelOptions(models = [], selectedModel = '') { + if (!modelSelect) return; + const modelIds = Array.isArray(models) ? models.filter(Boolean) : []; + modelSelect.innerHTML = ''; + + const noneOpt = document.createElement('option'); + noneOpt.value = ''; + noneOpt.textContent = 'None (omit model)'; + modelSelect.appendChild(noneOpt); + + if (selectedModel && !modelIds.includes(selectedModel)) { + const storedOpt = document.createElement('option'); + storedOpt.value = selectedModel; + storedOpt.textContent = `Stored: ${selectedModel}`; + modelSelect.appendChild(storedOpt); + } + + for (const id of modelIds) { + const opt = document.createElement('option'); + opt.value = id; + opt.textContent = id; + modelSelect.appendChild(opt); + } + + const hasSelected = [...modelSelect.options].some(opt => opt.value === selectedModel); + modelSelect.value = hasSelected ? selectedModel : ''; + } + + async function fetchModels(preferredModel = '') { + if (!modelSelect || !refreshModelsBtn) return; + const modelsUrl = AiClassifier.buildModelsUrl(endpointInput.value); + if (!modelsUrl) { + setModelHelp('Set a valid endpoint to load models.', true); + populateModelOptions([], preferredModel || modelSelect.value); + return; + } + + refreshModelsBtn.disabled = true; + setModelHelp('Loading models...'); + + try { + const response = await fetch(modelsUrl, { method: 'GET' }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + const data = await response.json(); + let models = []; + if (Array.isArray(data?.data)) { + models = data.data.map(model => model?.id ?? model?.name ?? model?.model ?? '').filter(Boolean); + } else if (Array.isArray(data?.models)) { + models = data.models.map(model => model?.id ?? model?.name ?? model?.model ?? '').filter(Boolean); + } else if (Array.isArray(data)) { + models = data.map(model => model?.id ?? model?.name ?? model?.model ?? model).filter(Boolean); + } + models = [...new Set(models)]; + populateModelOptions(models, preferredModel || modelSelect.value); + setModelHelp(models.length ? `Loaded ${models.length} model${models.length === 1 ? '' : 's'}.` : 'No models returned.'); + } catch (e) { + logger.aiLog('[options] failed to load models', { level: 'warn' }, e); + setModelHelp('Failed to load models. Check the endpoint and network.', true); + populateModelOptions([], preferredModel || modelSelect.value); + } finally { + refreshModelsBtn.disabled = false; + } + } + + populateModelOptions([], storedModel); + refreshModelsBtn?.addEventListener('click', () => { + fetchModels(modelSelect.value); + }); + const templates = { openai: browser.i18n.getMessage('template.openai'), qwen: browser.i18n.getMessage('template.qwen'), @@ -276,6 +359,7 @@ document.addEventListener('DOMContentLoaded', async () => { await loadErrors(); updateDiffDisplay(); + await fetchModels(storedModel); [htmlToggle, stripUrlToggle, altTextToggle, collapseWhitespaceToggle, tokenReductionToggle].forEach(toggle => { toggle.addEventListener('change', () => { @@ -914,6 +998,7 @@ document.addEventListener('DOMContentLoaded', async () => { document.getElementById('save').addEventListener('click', async () => { const endpoint = endpointInput.value.trim(); + const model = modelSelect?.value || ''; const templateName = templateSelect.value; const customTemplateText = customTemplate.value; const customSystemPrompt = systemBox.value; @@ -979,10 +1064,10 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.set({ endpoint, model, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); await applyTheme(theme); try { - await AiClassifier.setConfig({ endpoint, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ endpoint, model, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); From 1680ad6c3085a11a36dcb138103f1787367d64d1 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Fri, 30 Jan 2026 02:54:19 -0600 Subject: [PATCH 111/124] Add optional OpenAI auth headers --- AGENTS.md | 1 + README.md | 3 +++ background.js | 7 ++++-- manifest.json | 2 +- modules/AiClassifier.js | 28 ++++++++++++++++++++++- options/dataTransfer.js | 3 +++ options/options.html | 26 ++++++++++++++++++++++ options/options.js | 49 ++++++++++++++++++++++++++++++++++++++--- 8 files changed, 112 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f2ea2d9..aece578 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,7 @@ There are currently no automated tests for this project. If you add tests in the Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. The options page can query `/v1/models` from the same base URL to populate the Model dropdown; selecting **None** omits the `model` field from the request payload. +Advanced options allow an optional API key plus `OpenAI-Organization` and `OpenAI-Project` headers; these headers are only sent when values are provided. Responses are expected to include a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in the response text and ignores any surrounding commentary. ## Documentation diff --git a/README.md b/README.md index 957e5e0..5b908ea 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Configurable endpoint** – set the classification service base URL on the options page. - **Model selection** – load available models from the endpoint and choose one (or omit the model field). +- **Optional OpenAI auth headers** – provide an API key plus optional organization/project headers when needed. - **Prompt templates** – choose between OpenAI/ChatML, Qwen, Mistral, Harmony (gpt-oss), or provide your own custom template. - **Custom system prompts** – tailor the instructions sent to the model for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. @@ -82,6 +83,8 @@ Sortana is implemented entirely with standard WebExtension scripts—no custom e 1. Open the add-on's options and set the base URL of your classification service (Sortana will append `/v1/completions`). Use the Model dropdown to load `/v1/models` and select a model or choose **None** to omit the `model` field. + Advanced settings include optional API key, organization, and project headers + for OpenAI-hosted endpoints. 2. Use the **Classification Rules** section to add a criterion and optional actions such as tagging, moving, copying, forwarding, replying, deleting or archiving a message when it matches. Drag rules to diff --git a/background.js b/background.js index aef8cbb..827dec8 100644 --- a/background.js +++ b/background.js @@ -484,7 +484,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "model", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -514,10 +514,13 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } - if (changes.endpoint || changes.model || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { + if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { const config = {}; if (changes.endpoint) config.endpoint = changes.endpoint.newValue; if (changes.model) config.model = changes.model.newValue; + if (changes.apiKey) config.apiKey = changes.apiKey.newValue; + if (changes.openaiOrganization) config.openaiOrganization = changes.openaiOrganization.newValue; + if (changes.openaiProject) config.openaiProject = changes.openaiProject.newValue; if (changes.templateName) config.templateName = changes.templateName.newValue; if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; diff --git a/manifest.json b/manifest.json index a18e7cd..81baae5 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.4.0", + "version": "2.4.1", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index cb68382..b4c0907 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -40,6 +40,9 @@ let gTemplateText = ""; let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); let gModel = ""; +let gApiKey = ""; +let gOpenaiOrganization = ""; +let gOpenaiProject = ""; let gCache = new Map(); let gCacheLoaded = false; @@ -223,6 +226,15 @@ async function setConfig(config = {}) { if (typeof config.model === "string") { gModel = config.model.trim(); } + if (typeof config.apiKey === "string") { + gApiKey = config.apiKey.trim(); + } + if (typeof config.openaiOrganization === "string") { + gOpenaiOrganization = config.openaiOrganization.trim(); + } + if (typeof config.openaiProject === "string") { + gOpenaiProject = config.openaiProject.trim(); + } if (typeof config.debugLogging === "boolean") { setDebug(config.debugLogging); } @@ -241,6 +253,20 @@ async function setConfig(config = {}) { aiLog(`[AiClassifier] Template set to ${gTemplateName}`, {debug: true}); } +function buildAuthHeaders() { + const headers = {}; + if (gApiKey) { + headers.Authorization = `Bearer ${gApiKey}`; + } + if (gOpenaiOrganization) { + headers["OpenAI-Organization"] = gOpenaiOrganization; + } + if (gOpenaiProject) { + headers["OpenAI-Project"] = gOpenaiProject; + } + return headers; +} + function buildSystemPrompt() { return SYSTEM_PREFIX + (gCustomSystemPrompt || DEFAULT_CUSTOM_SYSTEM_PROMPT) + SYSTEM_SUFFIX; } @@ -453,7 +479,7 @@ async function classifyText(text, criterion, cacheKey = null) { try { const response = await fetch(gEndpoint, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", ...buildAuthHeaders() }, body: payload, }); diff --git a/options/dataTransfer.js b/options/dataTransfer.js index fdf096f..393b533 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -4,6 +4,9 @@ const KEY_GROUPS = { settings: [ 'endpoint', 'model', + 'apiKey', + 'openaiOrganization', + 'openaiProject', 'templateName', 'customTemplate', 'customSystemPrompt', diff --git a/options/options.html b/options/options.html index 4c1fd79..2a1431e 100644 --- a/options/options.html +++ b/options/options.html @@ -141,6 +141,32 @@
    -

    Built-in formats use native chat messages over Chat Completions. Custom format sends one templated user message.

    +

    OpenAI Chat uses native chat messages. Other formats send one templated user message over Chat Completions.

    - +
    -

    OpenAI Chat uses native chat messages. Other formats send one templated user message over Chat Completions.

    - +
    From 76195f9a92ef76e07264c036659a0366c5323494 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Sun, 6 Sep 2026 23:29:59 -0500 Subject: [PATCH 122/124] Migrate classifier to chat completions --- .codex | 0 AGENTS.md | 5 +-- README.md | 17 ++++---- _locales/en-US/messages.json | 8 ---- ai-filter.sln | 8 ---- background.js | 6 +-- manifest.json | 2 +- modules/AiClassifier.js | 55 ++++++------------------ options/dataTransfer.js | 2 - options/options.html | 17 -------- options/options.js | 83 ++++++++++++++++++++---------------- prompt_templates/harmony.txt | 21 --------- prompt_templates/mistral.txt | 12 ------ prompt_templates/openai.txt | 14 ------ prompt_templates/qwen.txt | 15 ------- 15 files changed, 73 insertions(+), 192 deletions(-) create mode 100644 .codex delete mode 100644 prompt_templates/harmony.txt delete mode 100644 prompt_templates/mistral.txt delete mode 100644 prompt_templates/openai.txt delete mode 100644 prompt_templates/qwen.txt diff --git a/.codex b/.codex new file mode 100644 index 0000000..e69de29 diff --git a/AGENTS.md b/AGENTS.md index 11876f7..1dcb02d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,6 @@ This file provides guidelines for codex agents contributing to the Sortana proje - `options/`: The options page HTML, JavaScript and bundled Bulma CSS (v1.0.3). - `details.html` and `details.js`: View AI reasoning and clear cache for a message. - `resources/`: Images and other static files. -- `prompt_templates/`: Prompt template files for the AI service (openai, qwen, mistral, harmony). - `build-xpi.ps1`: PowerShell script to package the extension. - `build-xpi.sh`: Bash script to package the extension. - `resources/svg2img.ps1`: PowerShell script to regenerate themed PNG icons from SVGs. @@ -35,10 +34,10 @@ There are currently no automated tests for this project. If you add tests in the ## Endpoint Notes -Sortana targets the `/v1/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. +Sortana targets the OpenAI-compatible `/v1/chat/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/chat/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. The options page can query `/v1/models` from the same base URL to populate the Model dropdown; selecting **None** omits the `model` field from the request payload. Advanced options allow an optional API key plus `OpenAI-Organization` and `OpenAI-Project` headers; these headers are only sent when values are provided. -Responses are expected to include a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in the response text and ignores any surrounding commentary. +Requests use `messages` with system and user roles. Responses are expected to include text in `choices[0].message.content` containing a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in that content and ignores any surrounding commentary. ## Documentation diff --git a/README.md b/README.md index fe67874..a0625d8 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,12 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filter rule. It allows you to classify email messages by sending their contents to a configurable -HTTP endpoint. Sortana uses the `/v1/completions` API; the options page stores a base -URL and appends `/v1/completions` when sending requests. The endpoint should respond -with JSON indicating whether the message meets a specified criterion, including a -short reasoning summary. -Responses are parsed by extracting the last JSON object in the response text and +HTTP endpoint. Sortana uses the OpenAI-compatible `/v1/chat/completions` API; the options page stores a base +URL and appends `/v1/chat/completions` when sending requests. Requests contain a system +message and a user message. The endpoint should respond +with text in `choices[0].message.content` containing JSON indicating whether the +message meets a specified criterion, including a short reasoning summary. +Responses are parsed by extracting the last JSON object in that content and expecting a `match` (or `matched`) boolean plus a `reason` string. ## Features @@ -16,10 +17,10 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Configurable endpoint** – set the classification service base URL on the options page. - **Model selection** – load available models from the endpoint and choose one (or omit the model field). - **Optional OpenAI auth headers** – provide an API key plus optional organization/project headers when needed. -- **Prompt templates** – choose between OpenAI/ChatML, Qwen, Mistral, Harmony (gpt-oss), or provide your own custom template. -- **Custom system prompts** – tailor the instructions sent to the model for more precise results. +- **Custom system prompts** – tailor the instructions sent as the system message for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. - **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. + Provider-specific advanced fields are forwarded as configured and may be rejected by strict OpenAI endpoints. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. - **Debug tab** – view the last request payload and a diff between the unaltered message text and the final prompt. @@ -83,7 +84,7 @@ Sortana is implemented entirely with documented MailExtension/WebExtension APIs. ## Usage 1. Open the add-on's options and set the base URL of your classification service - (Sortana will append `/v1/completions`). Use the Model dropdown to load + (Sortana will append `/v1/chat/completions`). Use the Model dropdown to load `/v1/models` and select a model or choose **None** to omit the `model` field. Advanced settings include optional API key, organization, and project headers for OpenAI-hosted endpoints. diff --git a/_locales/en-US/messages.json b/_locales/en-US/messages.json index 017e1a3..472956a 100644 --- a/_locales/en-US/messages.json +++ b/_locales/en-US/messages.json @@ -4,16 +4,8 @@ "doesntMatch": { "message": "doesn't match" }, "options.title": { "message": "AI Filter Options" }, "options.endpoint": { "message": "Endpoint" }, - "options.template": { "message": "Prompt template" }, - "options.customTemplate": { "message": "Custom template" }, "options.systemInstructions": { "message": "System instructions" }, "options.reset": { "message": "Reset to default" }, - "options.placeholders": { "message": "Placeholders: {{system}}, {{email}}, {{query}}" }, - "template.openai": { "message": "OpenAI / ChatML" }, - "template.qwen": { "message": "Qwen" }, - "template.mistral": { "message": "Mistral" }, - "template.harmony": { "message": "Harmony (gpt-oss)" }, - "template.custom": { "message": "Custom" }, "options.save": { "message": "Save" }, "options.debugLogging": { "message": "Enable debug logging" }, "options.htmlToMarkdown": { "message": "Convert HTML body to Markdown" }, diff --git a/ai-filter.sln b/ai-filter.sln index 7ded341..52e32fe 100644 --- a/ai-filter.sln +++ b/ai-filter.sln @@ -39,14 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "en-US", "en-US", "{8BEA7793 _locales\en-US\messages.json = _locales\en-US\messages.json EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "prompt_templates", "prompt_templates", "{86516D53-50D4-4FE2-9D8A-977A8F5EBDBD}" - ProjectSection(SolutionItems) = preProject - prompt_templates\mistral.txt = prompt_templates\mistral.txt - prompt_templates\openai.txt = prompt_templates\openai.txt - prompt_templates\qwen.txt = prompt_templates\qwen.txt - prompt_templates\harmony.txt = prompt_templates\harmony.txt - EndProjectSection -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "resources", "resources", "{68A87938-5C2B-49F5-8AAA-8A34FBBFD854}" ProjectSection(SolutionItems) = preProject resources\svg2img.ps1 = resources\svg2img.ps1 diff --git a/background.js b/background.js index b2efc89..5cfc112 100644 --- a/background.js +++ b/background.js @@ -484,7 +484,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "templateName", "customTemplate", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; @@ -514,15 +514,13 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } - if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.templateName || changes.customTemplate || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { + if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { const config = {}; if (changes.endpoint) config.endpoint = changes.endpoint.newValue; if (changes.model) config.model = changes.model.newValue; if (changes.apiKey) config.apiKey = changes.apiKey.newValue; if (changes.openaiOrganization) config.openaiOrganization = changes.openaiOrganization.newValue; if (changes.openaiProject) config.openaiProject = changes.openaiProject.newValue; - if (changes.templateName) config.templateName = changes.templateName.newValue; - if (changes.customTemplate) config.customTemplate = changes.customTemplate.newValue; if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; if (changes.aiParams) { config.aiParams = changes.aiParams.newValue; diff --git a/manifest.json b/manifest.json index 5dd0e72..139e60e 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "2.4.3", + "version": "3.0.0", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index dbbde92..583eec0 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -4,7 +4,7 @@ import { DEFAULT_AI_PARAMS } from "./defaultParams.js"; const storage = (globalThis.messenger ?? globalThis.browser).storage; -const COMPLETIONS_PATH = "/v1/completions"; +const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; const MODELS_PATH = "/v1/models"; const SYSTEM_PREFIX = `You are an email-classification assistant. @@ -22,10 +22,7 @@ Do not add any other keys, text, or formatting.`; let gEndpointBase = "http://127.0.0.1:5000"; let gEndpoint = buildEndpointUrl(gEndpointBase); -let gTemplateName = "openai"; -let gCustomTemplate = ""; let gCustomSystemPrompt = DEFAULT_CUSTOM_SYSTEM_PROMPT; -let gTemplateText = ""; let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); let gModel = ""; @@ -44,7 +41,7 @@ function normalizeEndpointBase(endpoint) { if (!base) { return ""; } - base = base.replace(/\/v1\/(completions|models)\/?$/i, ""); + base = base.replace(/\/v1\/(chat\/completions|completions|models)\/?$/i, ""); return base; } @@ -55,7 +52,7 @@ function buildEndpointUrl(endpointBase) { } const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; const needsSlash = withScheme.endsWith("/"); - const path = COMPLETIONS_PATH.replace(/^\//, ""); + const path = CHAT_COMPLETIONS_PATH.replace(/^\//, ""); return `${withScheme}${needsSlash ? "" : "/"}${path}`; } @@ -147,21 +144,6 @@ async function saveCache(updatedKey, updatedValue) { } -async function loadTemplate(name) { - try { - const url = typeof browser !== "undefined" && browser.runtime?.getURL - ? browser.runtime.getURL(`prompt_templates/${name}.txt`) - : `resource://aifilter/prompt_templates/${name}.txt`; - const res = await fetch(url); - if (res.ok) { - return await res.text(); - } - } catch (e) { - aiLog(`Failed to load template '${name}':`, {level: 'error'}, e); - } - return ""; -} - async function setConfig(config = {}) { if (typeof config.endpoint === "string") { const base = normalizeEndpointBase(config.endpoint); @@ -170,12 +152,6 @@ async function setConfig(config = {}) { } gEndpoint = buildEndpointUrl(gEndpointBase); } - if (config.templateName) { - gTemplateName = config.templateName; - } - if (typeof config.customTemplate === "string") { - gCustomTemplate = config.customTemplate; - } if (typeof config.customSystemPrompt === "string") { gCustomSystemPrompt = config.customSystemPrompt; } @@ -201,17 +177,11 @@ async function setConfig(config = {}) { if (typeof config.debugLogging === "boolean") { setDebug(config.debugLogging); } - if (gTemplateName === "custom") { - gTemplateText = gCustomTemplate; - } else { - gTemplateText = await loadTemplate(gTemplateName); - } if (!gEndpoint) { gEndpoint = buildEndpointUrl(gEndpointBase); } aiLog(`[AiClassifier] Endpoint base set to ${gEndpointBase}`, {debug: true}); aiLog(`[AiClassifier] Endpoint set to ${gEndpoint}`, {debug: true}); - aiLog(`[AiClassifier] Template set to ${gTemplateName}`, {debug: true}); } function buildAuthHeaders() { @@ -234,13 +204,7 @@ function buildSystemPrompt() { function buildPrompt(body, criterion) { aiLog(`[AiClassifier] Building prompt with criterion: "${criterion}"`, {debug: true}); - const data = { - system: buildSystemPrompt(), - email: body, - query: criterion, - }; - let template = gTemplateText || ""; - return template.replace(/{{\s*(\w+)\s*}}/g, (m, key) => data[key] || ""); + return `**Email Contents**\n\`\`\`\n${body}\n\`\`\`\nClassification Criterion: ${criterion}`; } function getCachedResult(cacheKey) { @@ -265,7 +229,10 @@ function getReason(cacheKey) { function buildPayload(text, criterion) { let payloadObj = Object.assign({ - prompt: buildPrompt(text, criterion) + messages: [ + { role: "system", content: buildSystemPrompt() }, + { role: "user", content: buildPrompt(text, criterion) } + ] }, gAiParams); if (gModel) { payloadObj.model = gModel; @@ -337,7 +304,11 @@ function extractLastJsonObject(text) { } function parseMatch(result) { - const rawText = result.choices?.[0]?.text || ""; + const rawText = result.choices?.[0]?.message?.content; + if (typeof rawText !== "string") { + reportParseError("Chat response missing text content.", JSON.stringify(result).slice(0, 800)); + return { matched: false, reason: "" }; + } const candidate = extractLastJsonObject(rawText); if (!candidate) { reportParseError("No JSON object found in AI response.", rawText.slice(0, 800)); diff --git a/options/dataTransfer.js b/options/dataTransfer.js index 393b533..2c0b26e 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -7,8 +7,6 @@ const KEY_GROUPS = { 'apiKey', 'openaiOrganization', 'openaiProject', - 'templateName', - 'customTemplate', 'customSystemPrompt', 'aiParams', 'debugLogging', diff --git a/options/options.html b/options/options.html index cfc370d..cd3d581 100644 --- a/options/options.html +++ b/options/options.html @@ -92,23 +92,6 @@

    -
    - -
    -
    - -
    -
    -
    - - -
    diff --git a/options/options.js b/options/options.js index 9a61542..2a8d3c4 100644 --- a/options/options.js +++ b/options/options.js @@ -7,8 +7,6 @@ document.addEventListener('DOMContentLoaded', async () => { const { DEFAULT_AI_PARAMS } = await import(browser.runtime.getURL('modules/defaultParams.js')); const defaults = await storage.local.get([ 'endpoint', - 'templateName', - 'customTemplate', 'customSystemPrompt', 'model', 'apiKey', @@ -163,21 +161,47 @@ document.addEventListener('DOMContentLoaded', async () => { async function fetchModels(preferredModel = '') { if (!modelSelect || !refreshModelsBtn) return; const modelsUrl = AiClassifier.buildModelsUrl(endpointInput.value); + const selectedModel = preferredModel || modelSelect.value; if (!modelsUrl) { + logger.aiLog('[options] model refresh skipped: invalid endpoint', { level: 'warn' }, { + endpoint: endpointInput.value + }); setModelHelp('Set a valid endpoint to load models.', true); - populateModelOptions([], preferredModel || modelSelect.value); + populateModelOptions([], selectedModel); return; } refreshModelsBtn.disabled = true; setModelHelp('Loading models...'); + const headers = buildAuthHeaders(); + logger.aiLog('[options] loading models', {}, { + endpoint: endpointInput.value, + modelsUrl, + selectedModel, + hasAuthorization: typeof headers.Authorization === 'string', + hasOrganization: typeof headers['OpenAI-Organization'] === 'string', + hasProject: typeof headers['OpenAI-Project'] === 'string' + }); try { - const response = await fetch(modelsUrl, { method: 'GET', headers: buildAuthHeaders() }); + const response = await fetch(modelsUrl, { method: 'GET', headers }); + logger.aiLog('[options] model refresh response received', {}, { + status: response.status, + ok: response.ok, + contentType: response.headers.get('content-type') || '' + }); if (!response.ok) { throw new Error(`HTTP ${response.status}`); } const data = await response.json(); + logger.aiLog('[options] model refresh payload parsed', {}, { + payloadType: Array.isArray(data) ? 'array' : typeof data, + hasDataArray: Array.isArray(data?.data), + hasModelsArray: Array.isArray(data?.models), + topLevelKeys: data && typeof data === 'object' && !Array.isArray(data) + ? Object.keys(data) + : [] + }); let models = []; if (Array.isArray(data?.data)) { models = data.data.map(model => model?.id ?? model?.name ?? model?.model ?? '').filter(Boolean); @@ -187,12 +211,23 @@ document.addEventListener('DOMContentLoaded', async () => { models = data.map(model => model?.id ?? model?.name ?? model?.model ?? model).filter(Boolean); } models = [...new Set(models)]; - populateModelOptions(models, preferredModel || modelSelect.value); + logger.aiLog('[options] model refresh parsed models', {}, { + count: models.length, + models + }); + populateModelOptions(models, selectedModel); setModelHelp(models.length ? `Loaded ${models.length} model${models.length === 1 ? '' : 's'}.` : 'No models returned.'); } catch (e) { logger.aiLog('[options] failed to load models', { level: 'warn' }, e); - setModelHelp('Failed to load models. Check the endpoint and network.', true); - populateModelOptions([], preferredModel || modelSelect.value); + const message = String(e?.message || e || ''); + const isLikelyCors = e instanceof TypeError || /Failed to fetch|NetworkError|Load failed/i.test(message); + setModelHelp( + isLikelyCors + ? 'Failed to load models. The server responded, but Thunderbird may have blocked access to the response. Check CORS on the AI server and verify the endpoint is reachable from the add-on.' + : 'Failed to load models. Check the endpoint, HTTP status, and network.', + true + ); + populateModelOptions([], selectedModel); } finally { refreshModelsBtn.disabled = false; } @@ -200,35 +235,10 @@ document.addEventListener('DOMContentLoaded', async () => { populateModelOptions([], storedModel); refreshModelsBtn?.addEventListener('click', () => { + logger.aiLog('[options] model refresh button clicked'); fetchModels(modelSelect.value); }); - const templates = { - openai: browser.i18n.getMessage('template.openai'), - qwen: browser.i18n.getMessage('template.qwen'), - mistral: browser.i18n.getMessage('template.mistral'), - harmony: browser.i18n.getMessage('template.harmony'), - custom: browser.i18n.getMessage('template.custom') - }; - const templateSelect = document.getElementById('template'); - for (const [value, label] of Object.entries(templates)) { - const opt = document.createElement('option'); - opt.value = value; - opt.textContent = label; - templateSelect.appendChild(opt); - } - templateSelect.value = defaults.templateName || 'openai'; - - const customBox = document.getElementById('custom-template-container'); - const customTemplate = document.getElementById('custom-template'); - customTemplate.value = defaults.customTemplate || ''; - - function updateVisibility() { - customBox.classList.toggle('is-hidden', templateSelect.value !== 'custom'); - } - templateSelect.addEventListener('change', updateVisibility); - updateVisibility(); - const advancedBox = document.getElementById('advanced-options'); const advancedBtn = document.getElementById('toggle-advanced'); advancedBtn.addEventListener('click', () => { @@ -1047,8 +1057,6 @@ document.addEventListener('DOMContentLoaded', async () => { const apiKey = apiKeyInput?.value.trim() || ''; const openaiOrganization = openaiOrgInput?.value.trim() || ''; const openaiProject = openaiProjectInput?.value.trim() || ''; - const templateName = templateSelect.value; - const customTemplateText = customTemplate.value; const customSystemPrompt = systemBox.value; const aiParamsSave = {}; for (const key of Object.keys(DEFAULT_AI_PARAMS)) { @@ -1112,10 +1120,11 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, model, apiKey, openaiOrganization, openaiProject, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.set({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.remove(['templateName', 'customTemplate']); await applyTheme(theme); try { - await AiClassifier.setConfig({ endpoint, model, apiKey, openaiOrganization, openaiProject, templateName, customTemplate: customTemplateText, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); diff --git a/prompt_templates/harmony.txt b/prompt_templates/harmony.txt deleted file mode 100644 index 5054543..0000000 --- a/prompt_templates/harmony.txt +++ /dev/null @@ -1,21 +0,0 @@ -<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. -Knowledge cutoff: 2024-06 -Current date: 2025-06-28 - -Reasoning: medium - -# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|> -<|start|>developer<|message|># Instructions - -{{system}}<|end|> -<|start|>user<|message|>**Email Contents** -``` -{{email}} -``` -Classification Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting.<|end|> -<|start|>assistant diff --git a/prompt_templates/mistral.txt b/prompt_templates/mistral.txt deleted file mode 100644 index 1ee56fa..0000000 --- a/prompt_templates/mistral.txt +++ /dev/null @@ -1,12 +0,0 @@ -[INST] {{system}} - -Email: -{{email}} - -Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting. -[/INST] diff --git a/prompt_templates/openai.txt b/prompt_templates/openai.txt deleted file mode 100644 index c770394..0000000 --- a/prompt_templates/openai.txt +++ /dev/null @@ -1,14 +0,0 @@ -<|im_start|>system -{{system}}<|im_end|> -<|im_start|>user -**Email Contents** -``` -{{email}} -``` -Classification Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting.<|im_end|> -<|im_start|>assistant diff --git a/prompt_templates/qwen.txt b/prompt_templates/qwen.txt deleted file mode 100644 index 1aa0ef0..0000000 --- a/prompt_templates/qwen.txt +++ /dev/null @@ -1,15 +0,0 @@ -<|im_start|>system -{{system}} -<|im_end|> -<|im_start|>user -Email: -{{email}} - -Criterion: {{query}} -Remember, return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting. -<|im_end|> -<|im_start|>assistant From 9a5404cb629df3c5808cdc812044ef91196008c0 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Wed, 7 Oct 2026 23:57:49 -0500 Subject: [PATCH 123/124] Add SystemOne classification support and release version 4.0.0 --- AGENTS.md | 59 ++- README.md | 104 +++++- background.js | 82 ++-- details.html | 27 ++ details.js | 34 +- manifest.json | 2 +- modules/AiClassifier.js | 559 +++++++--------------------- modules/backends/chatCompletions.js | 81 ++++ modules/backends/systemOne.js | 17 + modules/classificationConfig.js | 32 ++ options/dataTransfer.js | 8 +- options/options.html | 21 +- options/options.js | 131 ++++--- tests/cacheRestore.test.mjs | 84 +++++ tests/classification.test.mjs | 167 +++++++++ tests/ruleExecution.test.mjs | 80 ++++ 16 files changed, 916 insertions(+), 572 deletions(-) create mode 100644 modules/backends/chatCompletions.js create mode 100644 modules/backends/systemOne.js create mode 100644 modules/classificationConfig.js create mode 100644 tests/cacheRestore.test.mjs create mode 100644 tests/classification.test.mjs create mode 100644 tests/ruleExecution.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 1dcb02d..673e422 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ This file provides guidelines for codex agents contributing to the Sortana proje - `modules/`: Contains reusable JavaScript modules such as `AiClassifier.js`, `defaultParams.js` and `themeUtils.js`. - `options/`: The options page HTML, JavaScript and bundled Bulma CSS (v1.0.3). -- `details.html` and `details.js`: View AI reasoning and clear cache for a message. +- `details.html` and `details.js`: View classification evidence and clear cache for a message. - `resources/`: Images and other static files. - `build-xpi.ps1`: PowerShell script to package the extension. - `build-xpi.sh`: Bash script to package the extension. @@ -30,14 +30,42 @@ This file provides guidelines for codex agents contributing to the Sortana proje ## Testing -There are currently no automated tests for this project. If you add tests in the future, specify the commands to run them here. For now, verification must happen manually in Thunderbird. Do **not** run the `ps1` build script or the SVG processing script. +Run the dependency-free suite with Node.js 22 or newer: `node --test tests/*.test.mjs`. +The tests mock extension storage, HTTP responses, and Thunderbird rule execution. +Manual verification in Thunderbird is still required for UI and real-server integration. +Do **not** run the `ps1` build script or the SVG processing script. ## Endpoint Notes -Sortana targets the OpenAI-compatible `/v1/chat/completions` API. The endpoint value stored in settings is a base URL; the full request URL is constructed by appending `/v1/chat/completions` (adding a slash when needed) and defaulting to `https://` if no scheme is provided. -The options page can query `/v1/models` from the same base URL to populate the Model dropdown; selecting **None** omits the `model` field from the request payload. -Advanced options allow an optional API key plus `OpenAI-Organization` and `OpenAI-Project` headers; these headers are only sent when values are provided. -Requests use `messages` with system and user roles. Responses are expected to include text in `choices[0].message.content` containing a JSON object with `match` (or `matched`) plus a short `reason` string; the parser extracts the last JSON object in that content and ignores any surrounding commentary. +Sortana supports `apiType: "chat-completions"` (default) and `"systemone"`. +Plain adapter modules under `modules/backends/` own request construction and response +validation; `AiClassifier` owns shared transport, configuration snapshots, threshold +policy, and caching. The stored endpoint is a base URL. Normalization accepts `/v1` +and recognized full API paths, preserves proxy prefixes, and defaults to HTTPS +when no scheme is supplied. Model discovery uses `/v1/models`; **None** omits `model`. +Bearer authentication is optional for both APIs. Organization/project headers are +sent only for Chat Completions. + +Chat Completions sends system/user messages and extracts the last JSON object from +`choices[0].message.content`, requiring a `match` or `matched` boolean. SystemOne +sends prepared email text as `state` and a single `questions.match` of type `noul`. +It requires `answers.match.type === "noul"` and a finite numeric `noul` in `[0, 1]`. +No chat generation parameters or system prompt are sent to SystemOne. + +The global `decisionThreshold` defaults to 0.5; optional rule-level +`decisionThreshold` overrides it. UI percentages are stored as numbers in `[0, 1]`. +Matching uses `probability >= threshold` before display rounding. Details displays +probabilities in a progress bar with the percentage inside; threshold and model +appear in its tooltip. Results use the current threshold, never fabricated reasoning or action +history. No secondary model calls are made. + +`classify()` returns structured evidence; `classifyText()` remains a boolean wrapper. +Failures throw and are never cached. HTTP errors include the request URL and +JSON server error message when available; SystemOne 404 errors suggest checking +the running server build and decision-model support. The rule runner stops that email, reports the +error, and continues the next queued email. Snapshot settings and rules per email. +`inputCharacterLimit` replaces coupling input truncation to output `max_tokens`; +it applies with aggressive reduction and migrates from the old effective limit. ## Documentation @@ -71,10 +99,21 @@ text extracted from all text parts. ### Cache Strategy -`aiCache` persists classification results. Each key is the SHA‑256 hex of -`"|"` and maps to an object with `matched` and `reason` -properties. Any legacy `aiReasonCache` data is merged into `aiCache` the first -time the add-on loads after an update. +`aiCache` persists version 2 inference evidence. Keys are +`v2::`. +Include backend, endpoint, model, effective request parameters and preprocessing; +exclude secrets, actions and thresholds. Recompute SystemOne matches from cached +probability and current thresholds without automatically executing actions. +Entries retain a hashed message identifier for clearing across configurations. +Version 4.0.0 intentionally breaks legacy cache compatibility: legacy caches +(including `aiReasonCache`) are invalidated because their inference +configuration is unknown. Import/export validates versioned evidence; storage +changes refresh the running cache. Maintenance uses `sortana:getCacheSize` to +count validated entries in the background classifier, rather than raw storage keys. +Full restoration requires matching settings and rules as well as cache evidence; +`tests/cacheRestore.test.mjs` exercises export/reinstall/import through the actual +storage and Details handlers, including changed numeric Thunderbird message IDs. Clearing/importing during inference prevents +that in-flight result from repopulating the replaced cache. ### Icon Set Usage diff --git a/README.md b/README.md index a0625d8..b7fb0f5 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,11 @@ Sortana is an experimental Thunderbird add-on that integrates an AI-powered filter rule. It allows you to classify email messages by sending their contents to a configurable -HTTP endpoint. Sortana uses the OpenAI-compatible `/v1/chat/completions` API; the options page stores a base -URL and appends `/v1/chat/completions` when sending requests. Requests contain a system -message and a user message. The endpoint should respond -with text in `choices[0].message.content` containing JSON indicating whether the -message meets a specified criterion, including a short reasoning summary. -Responses are parsed by extracting the last JSON object in that content and -expecting a `match` (or `matched`) boolean plus a `reason` string. +HTTP endpoint. Choose **Chat Completions** for an OpenAI-compatible +`/v1/chat/completions` service or **SystemOne** for decision models served through +`/v1/systemone`. Chat Completions returns a match boolean and textual explanation; +SystemOne returns a match probability, compared against a configurable threshold. +The options page stores a base URL and shows the resolved request URL. ## Features @@ -19,7 +17,7 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Optional OpenAI auth headers** – provide an API key plus optional organization/project headers when needed. - **Custom system prompts** – tailor the instructions sent as the system message for more precise results. - **Persistent result caching** – classification results and reasoning are saved to disk so messages aren't re-evaluated across restarts. -- **Advanced parameters** – tune generation settings like temperature, top‑p and more from the options page. +- **Advanced parameters (Chat Completions)** – tune generation settings like temperature, top‑p and more from the options page. Provider-specific advanced fields are forwarded as configured and may be rejected by strict OpenAI endpoints. - **Markdown conversion** – optionally convert HTML bodies to Markdown before sending them to the AI service. - **Debug logging** – optional colorized logs help troubleshoot interactions with the AI service. @@ -33,7 +31,7 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. - **Status icons** – toolbar icons show when classification is in progress and briefly display success states. If a failure occurs the icon turns red briefly before returning to normal. - **Error notification** – failed classification displays a notification in Thunderbird. - **Session error log** – the Errors tab (visible only when errors occur) shows errors recorded since the last add-on start. -- **View reasoning** – inspect why rules matched via the Details popup. +- **View results** – inspect chat explanations or SystemOne probabilities and thresholds in the Details popup. - **Cache management** – clear cached results from the context menu or options page. - **Queue & timing stats** – monitor processing time on the Maintenance tab. - **Packaging scripts** – `build-xpi.ps1` (PowerShell) or `build-xpi.sh` (bash) build an XPI ready for installation. @@ -42,19 +40,91 @@ expecting a `match` (or `matched`) boolean plus a `reason` string. ### Cache Storage -Classification results are stored under the `aiCache` key in extension storage. -Each entry maps a SHA‑256 hash of `"|"` to an object -containing `matched` and `reason` fields. Older installations with a separate -`aiReasonCache` will be migrated automatically on startup. +Classification results are stored under `aiCache` using version 2 entries. Keys +combine a hashed Message-ID with a fingerprint of the criterion, backend, +endpoint, selected model, effective request settings, and preprocessing settings. +Credentials and thresholds are excluded. SystemOne probabilities are retained at +full precision; changing a threshold reuses the evidence without another request. +Changing inference settings requires a fresh classification. If the model behind +an unchanged server/model alias changes, clear the cache manually. + +**Version 4.0.0 upgrade note:** legacy cache entries and `aiReasonCache` are discarded because +they do not identify the inference configuration. Settings and rules are retained. +Imported legacy caches are also discarded. To restore cached Details after reinstalling, +export and import **Settings, Rules, and Cache** together: lookups require the +original criterion and matching inference settings. Import refreshes the running +classifier immediately; Maintenance counts its validated, loaded entries. A cache-only +import will not recreate rules or settings. Clearing a message's cache removes its +entries across rules and configurations. + +### SystemOne with llama-server + +Use a llama.cpp build supporting `/v1/systemone` and a compatible decision GGUF; +older builds and ordinary chat models cannot serve this API. A 404 with +`File Not Found` from `/v1/systemone` indicates the route is unavailable: check +the running server build and any reverse-proxy routing. Loading a decision GGUF +alone does not add this route to an older server. For example: + +```sh +llama-server -hf ggml-org/Laya-GGUF +``` + +1. Select **SystemOne** in Settings and set the endpoint to `http://127.0.0.1:8080` + (or your server's address). Refresh models and select one, or choose **None** + for a single loaded model. Router deployments require the appropriate model ID. +2. Set the default match probability threshold (initially **50%**). Each rule has + an optional percentage override; leave it blank to inherit the default. +3. Apply rules normally. One `noul` question is sent per evaluated rule. A rule + matches when the unrounded probability is **greater than or equal to** its + effective threshold. For example, 80% matches an 80% threshold. +4. Open Details to see a probability bar with the percentage inside. Hover over + the bar to see the current threshold and model. This is the + result under current settings, not an action history. Changing thresholds does + not automatically rerun actions. Decision models provide no textual explanation; + no second model is called to invent one. Use Chat Completions for explanations. + +Probabilities are model estimates, not guaranteed accuracy. Evaluate thresholds +against representative labeled emails, including long messages and attachments. +SystemOne uses the existing text extraction and attachment summaries; it does not +send attachment binaries or images. Chat prompts and generation settings remain +saved but are inactive for SystemOne. Optional Bearer authentication works with +both APIs; organization/project headers apply only to Chat Completions. + +**Input character limit** applies when aggressive reduction is enabled, independently +of maximum output tokens. Existing installations inherit their previous effective +character limit; the default is 3686. Character counts do not guarantee that input +fits a model's token context or llama-server batch limits. Leave room for question +instructions and configure the server for the chosen model. + +A request or response-parsing failure stops the remaining rules for that email, +records an error, and allows the queue to continue with the next email. Failed +results are never cached. Actions already completed are not rolled back. + +Upstream: [llama-server SystemOne documentation](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#typesafe-compatible-api-endpoints). + +### Verification + +Run the dependency-free tests with Node.js 22 or newer: + +```sh +node --test tests/*.test.mjs +``` + +Tests cover adapters, thresholds, URLs, settings validation, cache migration and +reuse, request failures, per-email snapshots, and queue behavior with mocked +Thunderbird services. Manual acceptance testing in Thunderbird should verify both +API types, model discovery, rule overrides and ordering, import/export, cache +clearing, Details, and stop-after-match against a running server. ## Architecture Overview Sortana is implemented entirely with documented MailExtension/WebExtension APIs. No custom experiment code or Thunderbird-internal fallback paths are required: - `background.js` loads saved settings, manages the classification queue and listens for new messages. -- `modules/AiClassifier.js` implements the classification logic and cache handling. +- `modules/AiClassifier.js` shares transport, threshold policy, and cache handling; + `modules/backends/` contains the Chat Completions and SystemOne adapters. - `options/` contains the HTML and JavaScript for configuring the endpoint and rules. -- `details.html` / `details.js` present cached reasoning for a message. +- `details.html` / `details.js` present cached classification evidence for a message. - `_locales/` holds localized strings used throughout the UI. ### Key Files @@ -65,7 +135,7 @@ Sortana is implemented entirely with documented MailExtension/WebExtension APIs. | `background.js` | Startup tasks and classification queue management. | | `modules/AiClassifier.js` | Core classification logic and cache handling. | | `options/options.html` and `options.js` | Endpoint and rule configuration UI. | -| `details.html` and `details.js` | View stored reasoning for a message. | +| `details.html` and `details.js` | View stored classification evidence for a message. | | `logger.js` | Colorized logging with optional debug mode. | ## Building @@ -84,7 +154,7 @@ Sortana is implemented entirely with documented MailExtension/WebExtension APIs. ## Usage 1. Open the add-on's options and set the base URL of your classification service - (Sortana will append `/v1/chat/completions`). Use the Model dropdown to load + (Sortana appends the path for the selected API type). Use the Model dropdown to load `/v1/models` and select a model or choose **None** to omit the `model` field. Advanced settings include optional API key, organization, and project headers for OpenAI-hosted endpoints. diff --git a/background.js b/background.js index 5cfc112..1b9f554 100644 --- a/background.js +++ b/background.js @@ -28,7 +28,6 @@ let stripUrlParams = false; let altTextImages = false; let collapseWhitespace = false; let tokenReduction = false; -let maxTokens = 4096; let TurndownService = null; let userTheme = 'auto'; let currentTheme = 'light'; @@ -53,6 +52,7 @@ function normalizeRules(rules) { if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); if (r.copyTarget || r.copyTo) actions.push({ type: 'copy', copyTarget: r.copyTarget || r.copyTo }); const rule = { criterion: r.criterion, actions }; + if (r.decisionThreshold !== undefined) rule.decisionThreshold = r.decisionThreshold; if (r.stopProcessing) rule.stopProcessing = true; if (r.unreadOnly) rule.unreadOnly = true; if (typeof r.minAgeDays === 'number') rule.minAgeDays = r.minAgeDays; @@ -318,10 +318,12 @@ async function processMessage(id) { updateActionIcon(); try { const full = await messenger.messages.getFull(id); + const snapshot = AiClassifier.getConfig(); + const rules = structuredClone(aiRules); const originalText = buildEmailText(full, false); let text = buildEmailText(full); - if (tokenReduction && maxTokens > 0) { - const limit = Math.floor(maxTokens * 0.9); + if (tokenReduction) { + const limit = snapshot.inputCharacterLimit; if (text.length > limit) { text = text.slice(0, limit); } @@ -345,7 +347,7 @@ async function processMessage(id) { identityId = null; } - for (const rule of aiRules) { + for (const rule of rules) { if (rule.enabled === false) { continue; } @@ -372,8 +374,8 @@ async function processMessage(id) { } } } - const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion); - const matched = await AiClassifier.classifyText(text, rule.criterion, cacheKey); + const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion, snapshot); + const { matched } = await AiClassifier.classify(text, rule.criterion, { cacheKey, snapshot, threshold: rule.decisionThreshold ?? snapshot.decisionThreshold }); if (matched) { for (const act of (rule.actions || [])) { if (act.type === 'tag' && act.tagKey) { @@ -456,18 +458,8 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(stored); } - const keys = []; - for (const msg of ids) { - const id = msg?.id ?? msg; - for (const rule of aiRules) { - const key = await AiClassifier.buildCacheKey(id, rule.criterion); - keys.push(key); - } - } - if (keys.length) { - await AiClassifier.removeCacheEntries(keys); - showTransientIcon(ICONS.circle); - } + for (const msg of ids) await AiClassifier.clearCacheForMessage(msg?.id ?? msg); + showTransientIcon(ICONS.circle); } (async () => { @@ -484,8 +476,13 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["apiType", "decisionThreshold", "inputCharacterLimit", "endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); + const { migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); + if (store.inputCharacterLimit === undefined) { + store.inputCharacterLimit = migratedInputLimit(store); + await storage.local.set({ inputCharacterLimit: store.inputCharacterLimit }); + } await AiClassifier.setConfig(store); userTheme = store.theme || 'auto'; currentTheme = userTheme === 'auto' ? await detectSystemTheme() : userTheme; @@ -495,9 +492,6 @@ async function clearCacheForMessages(idsInput) { altTextImages = store.altTextImages === true; collapseWhitespace = store.collapseWhitespace === true; tokenReduction = store.tokenReduction === true; - if (store.aiParams && typeof store.aiParams.max_tokens !== 'undefined') { - maxTokens = parseInt(store.aiParams.max_tokens) || maxTokens; - } showDebugTab = store.showDebugTab === true; const savedStats = await storage.local.get('classifyStats'); if (savedStats.classifyStats && typeof savedStats.classifyStats === 'object') { @@ -514,27 +508,9 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(newRules); logger.aiLog("aiRules updated from storage change", { debug: true }, aiRules); } - if (changes.endpoint || changes.model || changes.apiKey || changes.openaiOrganization || changes.openaiProject || changes.customSystemPrompt || changes.aiParams || changes.debugLogging) { - const config = {}; - if (changes.endpoint) config.endpoint = changes.endpoint.newValue; - if (changes.model) config.model = changes.model.newValue; - if (changes.apiKey) config.apiKey = changes.apiKey.newValue; - if (changes.openaiOrganization) config.openaiOrganization = changes.openaiOrganization.newValue; - if (changes.openaiProject) config.openaiProject = changes.openaiProject.newValue; - if (changes.customSystemPrompt) config.customSystemPrompt = changes.customSystemPrompt.newValue; - if (changes.aiParams) { - config.aiParams = changes.aiParams.newValue; - if (changes.aiParams.newValue && typeof changes.aiParams.newValue.max_tokens !== 'undefined') { - maxTokens = parseInt(changes.aiParams.newValue.max_tokens) || maxTokens; - } - } - if (changes.debugLogging) { - config.debugLogging = changes.debugLogging.newValue === true; - logger.setDebug(config.debugLogging); - } - await AiClassifier.setConfig(config); - logger.aiLog("AiClassifier config updated from storage change", { debug: true }, config); - } + const configChanges = Object.fromEntries(Object.entries(changes).map(([key, change]) => [key, change.newValue])); + AiClassifier.setConfig(configChanges); + if (changes.aiCache) AiClassifier.replaceCache(changes.aiCache.newValue); if (changes.htmlToMarkdown) { htmlToMarkdown = changes.htmlToMarkdown.newValue === true; logger.aiLog("htmlToMarkdown updated from storage change", { debug: true }, htmlToMarkdown); @@ -666,10 +642,10 @@ async function clearCacheForMessages(idsInput) { logger.aiLog("sortana:test – criterion", { debug: true }, criterion); try { - logger.aiLog("Calling AiClassifier.classifyText()", { debug: true }); - const result = await AiClassifier.classifyText(text, criterion); + logger.aiLog("Calling AiClassifier.classify()", { debug: true }); + const result = await AiClassifier.classify(text, criterion, { threshold: msg.decisionThreshold ?? AiClassifier.getConfig().decisionThreshold }); logger.aiLog("classify() returned", { debug: true }, result); - return { match: result }; + return { ...result, match: result.matched }; } catch (err) { logger.aiLog("Error in classify()", { level: 'error' }, err); @@ -716,13 +692,11 @@ async function clearCacheForMessages(idsInput) { aiRules = normalizeRules(stored); } const results = []; - for (const rule of aiRules) { - const key = await AiClassifier.buildCacheKey(id, rule.criterion); - const matched = AiClassifier.getCachedResult(key); - const reason = AiClassifier.getReason(key); - if (matched !== null || reason) { - results.push({ criterion: rule.criterion, matched, reason }); - } + const snapshot = AiClassifier.getConfig(); + for (const rule of structuredClone(aiRules)) { + const key = await AiClassifier.buildCacheKey(id, rule.criterion, snapshot); + const detail = AiClassifier.getCachedDetails(key, rule.decisionThreshold ?? snapshot.decisionThreshold); + if (detail) results.push({ criterion: rule.criterion, ...detail }); } return { subject, results }; } catch (e) { @@ -757,6 +731,8 @@ async function clearCacheForMessages(idsInput) { timingStats.last = typeof last === 'number' ? last : -1; await storage.local.set({ classifyStats: timingStats }); return { ok: true }; + } else if (msg?.type === "sortana:getCacheSize") { + return { count: await AiClassifier.getCacheSize() }; } else if (msg?.type === "sortana:getQueueCount") { return { count: queuedCount + (processing ? 1 : 0) }; } else if (msg?.type === "sortana:getErrorLog") { diff --git a/details.html b/details.html index d15a3c9..e84bf2e 100644 --- a/details.html +++ b/details.html @@ -4,6 +4,33 @@ AI Details +
    diff --git a/details.js b/details.js index f306e01..a4534d3 100644 --- a/details.js +++ b/details.js @@ -51,15 +51,39 @@ async function loadMessage(id) { article.className = `message ${color} mb-4`; const header = document.createElement('div'); header.className = 'message-header'; - header.innerHTML = `

    ${r.criterion}

    `; + const criterion = document.createElement('p'); + criterion.textContent = r.criterion; + header.appendChild(criterion); const body = document.createElement('div'); body.className = 'message-body'; const status = document.createElement('p'); status.textContent = r.matched ? 'Matched' : 'Did not match'; - const pre = document.createElement('pre'); - pre.textContent = r.reason || ''; - body.appendChild(status); - body.appendChild(pre); + if (r.backend === 'systemone') { + status.textContent += ' under current threshold'; + status.classList.add('mb-2'); + body.appendChild(status); + const percentage = r.probability * 100; + const bar = document.createElement('div'); + bar.className = `probability-bar ${color}`; + bar.setAttribute('role', 'progressbar'); + bar.setAttribute('aria-label', 'Match probability'); + bar.setAttribute('aria-valuemin', '0'); + bar.setAttribute('aria-valuemax', '100'); + bar.setAttribute('aria-valuenow', String(percentage)); + bar.title = `Threshold: ${r.threshold * 100}%${r.model ? ` · Model: ${r.model}` : ''}`; + const fill = document.createElement('div'); + fill.className = 'probability-fill'; + fill.style.width = `${percentage}%`; + const value = document.createElement('span'); + value.className = 'probability-value'; + value.textContent = `${percentage.toFixed(2)}%`; + bar.append(fill, value); + body.appendChild(bar); + } else { + const pre = document.createElement('pre'); + pre.textContent = r.reason || ''; + body.append(status, pre); + } article.appendChild(header); article.appendChild(body); container.appendChild(article); diff --git a/manifest.json b/manifest.json index 139e60e..e9a3088 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "3.0.0", + "version": "4.0.0", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 583eec0..51951e6 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -1,438 +1,151 @@ "use strict"; -import { aiLog, setDebug } from "../logger.js"; -import { DEFAULT_AI_PARAMS } from "./defaultParams.js"; - +import { aiLog, setDebug } from '../logger.js'; +import { CONFIG_DEFAULTS, validateSettings, validateThreshold } from './classificationConfig.js'; +import * as chat from './backends/chatCompletions.js'; +import * as systemone from './backends/systemOne.js'; const storage = (globalThis.messenger ?? globalThis.browser).storage; +const adapters = { 'chat-completions': chat, systemone }; +let config = structuredClone(CONFIG_DEFAULTS); +let cache = new Map(); +let loaded = false; +let loading; +let generation = 0; -const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; -const MODELS_PATH = "/v1/models"; - -const SYSTEM_PREFIX = `You are an email-classification assistant. -Read the email below and the classification criterion provided by the user. -`; - -const DEFAULT_CUSTOM_SYSTEM_PROMPT = "Determine whether the email satisfies the user's criterion."; - -const SYSTEM_SUFFIX = ` -Return ONLY a JSON object on a single line of the form: -{"match": true, "reason": ""} - if the email satisfies the criterion -{"match": false, "reason": ""} - otherwise - -Do not add any other keys, text, or formatting.`; - -let gEndpointBase = "http://127.0.0.1:5000"; -let gEndpoint = buildEndpointUrl(gEndpointBase); -let gCustomSystemPrompt = DEFAULT_CUSTOM_SYSTEM_PROMPT; - -let gAiParams = Object.assign({}, DEFAULT_AI_PARAMS); -let gModel = ""; -let gApiKey = ""; -let gOpenaiOrganization = ""; -let gOpenaiProject = ""; - -let gCache = new Map(); -let gCacheLoaded = false; - -function normalizeEndpointBase(endpoint) { - if (typeof endpoint !== "string") { - return ""; - } - let base = endpoint.trim(); - if (!base) { - return ""; - } - base = base.replace(/\/v1\/(chat\/completions|completions|models)\/?$/i, ""); - return base; +export function normalizeEndpointBase(endpoint) { + if (typeof endpoint !== 'string' || !endpoint.trim()) return ''; + return endpoint.trim().replace(/\/+$/, '').replace(/\/v1(?:\/(?:chat\/completions|completions|models|systemone))?$/i, ''); } - -function buildEndpointUrl(endpointBase) { - const base = normalizeEndpointBase(endpointBase); - if (!base) { - return ""; - } - const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; - const needsSlash = withScheme.endsWith("/"); - const path = CHAT_COMPLETIONS_PATH.replace(/^\//, ""); - return `${withScheme}${needsSlash ? "" : "/"}${path}`; +function urlFor(endpoint, path) { + const base = normalizeEndpointBase(endpoint); + return base ? `${/^https?:\/\//i.test(base) ? base : `https://${base}`}${path}` : ''; } - -function buildModelsUrl(endpointBase) { - const base = normalizeEndpointBase(endpointBase); - if (!base) { - return ""; - } - const withScheme = /^https?:\/\//i.test(base) ? base : `https://${base}`; - const needsSlash = withScheme.endsWith("/"); - const path = MODELS_PATH.replace(/^\//, ""); - return `${withScheme}${needsSlash ? "" : "/"}${path}`; +export function buildEndpointUrl(endpoint, apiType = 'chat-completions') { + if (!adapters[apiType]) throw new Error('Unknown API type.'); + return urlFor(endpoint, adapters[apiType].path); } - -async function sha256Hex(str) { - try { - const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(str)); - return Array.from(new Uint8Array(buf), b => b.toString(16).padStart(2, "0")).join(""); - } catch (e) { - aiLog("sha256Hex failed", { level: "error" }, e); - return ""; +export function buildModelsUrl(endpoint) { return urlFor(endpoint, '/v1/models'); } +export function getConfig() { return structuredClone(config); } +export function setConfig(update = {}) { + validateSettings(update); + for (const key of Object.keys(CONFIG_DEFAULTS)) { + if (update[key] !== undefined) config[key] = structuredClone(update[key]); } + config.aiParams = { ...CONFIG_DEFAULTS.aiParams, ...config.aiParams }; + for (const key of ['model', 'apiKey', 'openaiOrganization', 'openaiProject']) config[key] = String(config[key]).trim(); + setDebug(config.debugLogging); } - -async function resolveHeaderId(id) { - if (typeof id === "number" && typeof messenger?.messages?.get === "function") { +async function hash(value) { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(value)); + return Array.from(new Uint8Array(digest), b => b.toString(16).padStart(2, '0')).join(''); +} +async function messageHash(id) { + const api = globalThis.messenger ?? globalThis.browser; + if (typeof id === 'number' && api.messages?.get) { + const header = await api.messages.get(id); + id = header.headerMessageId || id; + } + return hash(String(id)); +} +function stable(value) { + if (Array.isArray(value)) return value.map(stable); + if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(k => [k, stable(value[k])])); + return value; +} +export async function buildCacheKey(id, criterion, snapshot = getConfig()) { + const fingerprint = { + backend: snapshot.apiType, endpoint: buildEndpointUrl(snapshot.endpoint, snapshot.apiType), + request: adapters[snapshot.apiType].buildRequest('', criterion, snapshot), + organization: snapshot.apiType === 'chat-completions' ? snapshot.openaiOrganization : '', + project: snapshot.apiType === 'chat-completions' ? snapshot.openaiProject : '', + preprocessing: Object.fromEntries(['htmlToMarkdown', 'stripUrlParams', 'altTextImages', 'collapseWhitespace', 'tokenReduction', 'inputCharacterLimit'].map(k => [k, snapshot[k]])) + }; + return `v2:${await messageHash(id)}:${await hash(JSON.stringify(stable(fingerprint)))}`; +} +export function validCacheEntry(key, entry) { + return /^v2:[a-f0-9]{64}:[a-f0-9]{64}$/.test(key) && entry?.version === 2 && + entry.messageHash === key.split(':')[1] && typeof entry.model === 'string' && + (entry.backend === 'systemone' + ? typeof entry.probability === 'number' && Number.isFinite(entry.probability) && entry.probability >= 0 && entry.probability <= 1 && entry.reason === null + : entry.backend === 'chat-completions' && typeof entry.matched === 'boolean' && entry.probability === null && (entry.reason === null || typeof entry.reason === 'string')); +} +export function replaceCache(data = {}) { + cache = new Map(Object.entries(data || {}).filter(([key, entry]) => validCacheEntry(key, entry))); + loaded = true; + generation++; +} +async function saveCache() { await storage.local.set({ aiCache: Object.fromEntries(cache) }); } +export async function init() { + if (loaded) return; + if (!loading) loading = (async () => { + const { aiCache = {}, aiReasonCache } = await storage.local.get(['aiCache', 'aiReasonCache']); + replaceCache(aiCache); + if (Object.keys(aiCache || {}).length !== cache.size) await saveCache(); + if (aiReasonCache !== undefined) await storage.local.remove('aiReasonCache'); + })().finally(() => { loading = null; }); + await loading; +} +function withPolicy(entry, threshold) { + const isDecision = entry.backend === 'systemone'; + if (isDecision) validateThreshold(threshold); + return { matched: isDecision ? entry.probability >= threshold : entry.matched, + reason: entry.reason, probability: entry.probability, threshold: isDecision ? threshold : null, + backend: entry.backend, model: entry.model }; +} +export function getCachedDetails(key, threshold = config.decisionThreshold) { + const entry = cache.get(key); + return entry ? withPolicy(entry, threshold) : null; +} +export function getCachedResult(key, threshold) { return getCachedDetails(key, threshold)?.matched ?? null; } +export function getReason(key) { return cache.get(key)?.reason ?? null; } +export async function classify(text, criterion, { cacheKey = null, snapshot = getConfig(), threshold = snapshot.decisionThreshold } = {}) { + await init(); + if (snapshot.apiType === 'systemone') validateThreshold(threshold); + const cached = getCachedDetails(cacheKey, threshold); + if (cached) return cached; + const epoch = generation; + const adapter = adapters[snapshot.apiType]; + const payload = adapter.buildRequest(text, criterion, snapshot); + try { await storage.local.set({ lastPayload: payload }); } + catch (error) { aiLog('Failed to save last payload', { level: 'warn' }, error); } + const headers = { 'Content-Type': 'application/json' }; + if (snapshot.apiKey) headers.Authorization = `Bearer ${snapshot.apiKey}`; + if (snapshot.apiType === 'chat-completions') { + if (snapshot.openaiOrganization) headers['OpenAI-Organization'] = snapshot.openaiOrganization; + if (snapshot.openaiProject) headers['OpenAI-Project'] = snapshot.openaiProject; + } + const endpoint = buildEndpointUrl(snapshot.endpoint, snapshot.apiType); + const response = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify(payload) }); + if (!response.ok) { + let detail = ''; try { - const hdr = await messenger.messages.get(id); - if (hdr?.headerMessageId) { - return hdr.headerMessageId; - } - } catch (e) { - aiLog(`Failed to resolve headerMessageId for ${id}`, { level: 'warn' }, e); - } + const body = await response.json(); + const message = body?.error?.message; + if (typeof message === 'string') detail = ` Server response: ${message.slice(0, 300)}.`; + } catch { /* Non-JSON error responses still report the URL and status. */ } + const hint = response.status === 404 && snapshot.apiType === 'systemone' + ? ' Verify that the running server build supports /v1/systemone and the selected decision model.' : ''; + throw new Error(`Classification request failed: HTTP ${response.status} at ${endpoint}.${detail}${hint}`); } - return String(id); -} - -async function buildCacheKey(id, criterion) { - const resolvedId = await resolveHeaderId(id); - return sha256Hex(`${resolvedId}|${criterion}`); -} - -async function loadCache() { - if (gCacheLoaded) { - return; - } - aiLog(`[AiClassifier] Loading cache`, {debug: true}); - try { - const { aiCache, aiReasonCache } = await storage.local.get(["aiCache", "aiReasonCache"]); - if (aiCache) { - for (let [k, v] of Object.entries(aiCache)) { - if (v && typeof v === "object") { - gCache.set(k, { matched: v.matched ?? null, reason: v.reason || "" }); - } else { - gCache.set(k, { matched: v, reason: "" }); - } - } - aiLog(`[AiClassifier] Loaded ${gCache.size} cache entries`, {debug: true}); - } else { - aiLog(`[AiClassifier] Cache is empty`, {debug: true}); - } - if (aiReasonCache) { - aiLog(`[AiClassifier] Migrating ${Object.keys(aiReasonCache).length} reason entries`, {debug: true}); - for (let [k, reason] of Object.entries(aiReasonCache)) { - let entry = gCache.get(k) || { matched: null, reason: "" }; - entry.reason = reason; - gCache.set(k, entry); - } - await storage.local.remove("aiReasonCache"); - await storage.local.set({ aiCache: Object.fromEntries(gCache) }); - } - } catch (e) { - aiLog(`Failed to load cache`, {level: 'error'}, e); - } - gCacheLoaded = true; -} - - -async function saveCache(updatedKey, updatedValue) { - if (typeof updatedKey !== "undefined") { - aiLog(`[AiClassifier] ⮡ Persisting entry '${updatedKey}' → ${updatedValue}`, {debug: true}); - } - try { - await storage.local.set({ aiCache: Object.fromEntries(gCache) }); - } catch (e) { - aiLog(`Failed to save cache`, {level: 'error'}, e); - } -} - - -async function setConfig(config = {}) { - if (typeof config.endpoint === "string") { - const base = normalizeEndpointBase(config.endpoint); - if (base) { - gEndpointBase = base; - } - gEndpoint = buildEndpointUrl(gEndpointBase); - } - if (typeof config.customSystemPrompt === "string") { - gCustomSystemPrompt = config.customSystemPrompt; - } - if (config.aiParams && typeof config.aiParams === "object") { - for (let [k, v] of Object.entries(config.aiParams)) { - if (k in gAiParams && typeof v !== "undefined") { - gAiParams[k] = v; - } - } - } - if (typeof config.model === "string") { - gModel = config.model.trim(); - } - if (typeof config.apiKey === "string") { - gApiKey = config.apiKey.trim(); - } - if (typeof config.openaiOrganization === "string") { - gOpenaiOrganization = config.openaiOrganization.trim(); - } - if (typeof config.openaiProject === "string") { - gOpenaiProject = config.openaiProject.trim(); - } - if (typeof config.debugLogging === "boolean") { - setDebug(config.debugLogging); - } - if (!gEndpoint) { - gEndpoint = buildEndpointUrl(gEndpointBase); - } - aiLog(`[AiClassifier] Endpoint base set to ${gEndpointBase}`, {debug: true}); - aiLog(`[AiClassifier] Endpoint set to ${gEndpoint}`, {debug: true}); -} - -function buildAuthHeaders() { - const headers = {}; - if (gApiKey) { - headers.Authorization = `Bearer ${gApiKey}`; - } - if (gOpenaiOrganization) { - headers["OpenAI-Organization"] = gOpenaiOrganization; - } - if (gOpenaiProject) { - headers["OpenAI-Project"] = gOpenaiProject; - } - return headers; -} - -function buildSystemPrompt() { - return SYSTEM_PREFIX + (gCustomSystemPrompt || DEFAULT_CUSTOM_SYSTEM_PROMPT) + SYSTEM_SUFFIX; -} - -function buildPrompt(body, criterion) { - aiLog(`[AiClassifier] Building prompt with criterion: "${criterion}"`, {debug: true}); - return `**Email Contents**\n\`\`\`\n${body}\n\`\`\`\nClassification Criterion: ${criterion}`; -} - -function getCachedResult(cacheKey) { - if (!gCacheLoaded) { - return null; - } - if (cacheKey && gCache.has(cacheKey)) { - aiLog(`[AiClassifier] Cache hit for key: ${cacheKey}`, {debug: true}); - const entry = gCache.get(cacheKey); - return entry?.matched ?? null; - } - return null; -} - -function getReason(cacheKey) { - if (!gCacheLoaded) { - return null; - } - const entry = gCache.get(cacheKey); - return cacheKey && entry ? entry.reason || null : null; -} - -function buildPayload(text, criterion) { - let payloadObj = Object.assign({ - messages: [ - { role: "system", content: buildSystemPrompt() }, - { role: "user", content: buildPrompt(text, criterion) } - ] - }, gAiParams); - if (gModel) { - payloadObj.model = gModel; - } - return JSON.stringify(payloadObj); -} - -function reportParseError(message, detail) { - try { - const runtime = (globalThis.browser ?? globalThis.messenger)?.runtime; - if (!runtime?.sendMessage) { - return; - } - runtime.sendMessage({ - type: "sortana:recordError", - context: "AI response parsing", - message, - detail - }).catch(() => {}); - } catch (e) { - aiLog("Failed to report parse error", { level: "warn" }, e); - } -} - -function extractLastJsonObject(text) { - let last = null; - let start = -1; - let depth = 0; - let inString = false; - let escape = false; - - for (let i = 0; i < text.length; i += 1) { - const ch = text[i]; - if (inString) { - if (escape) { - escape = false; - continue; - } - if (ch === "\\") { - escape = true; - continue; - } - if (ch === "\"") { - inString = false; - } - continue; - } - if (ch === "\"") { - inString = true; - continue; - } - if (ch === "{") { - if (depth === 0) { - start = i; - } - depth += 1; - continue; - } - if (ch === "}" && depth > 0) { - depth -= 1; - if (depth === 0 && start !== -1) { - last = text.slice(start, i + 1); - start = -1; - } - } - } - - return last; -} - -function parseMatch(result) { - const rawText = result.choices?.[0]?.message?.content; - if (typeof rawText !== "string") { - reportParseError("Chat response missing text content.", JSON.stringify(result).slice(0, 800)); - return { matched: false, reason: "" }; - } - const candidate = extractLastJsonObject(rawText); - if (!candidate) { - reportParseError("No JSON object found in AI response.", rawText.slice(0, 800)); - return { matched: false, reason: "" }; - } - - let obj; - try { - obj = JSON.parse(candidate); - } catch (e) { - reportParseError("Failed to parse JSON from AI response.", candidate.slice(0, 800)); - return { matched: false, reason: "" }; - } - - const matchValue = Object.prototype.hasOwnProperty.call(obj, "match") ? obj.match : obj.matched; - const matched = matchValue === true; - if (matchValue !== true && matchValue !== false) { - reportParseError("AI response missing valid match boolean.", candidate.slice(0, 800)); - } - - const reasonValue = obj.reason ?? obj.reasoning ?? obj.explaination; - const reason = typeof reasonValue === "string" ? reasonValue : ""; - - return { matched, reason }; -} - -function cacheEntry(cacheKey, matched, reason) { - if (!cacheKey) { - return; - } - aiLog(`[AiClassifier] Caching entry '${cacheKey}'`, {debug: true}); - const entry = gCache.get(cacheKey) || { matched: null, reason: "" }; - if (typeof matched === "boolean") { - entry.matched = matched; - } - if (typeof reason === "string") { - entry.reason = reason; - } - gCache.set(cacheKey, entry); - saveCache(cacheKey, entry); -} - -async function removeCacheEntries(keys = []) { - if (!Array.isArray(keys)) { - keys = [keys]; - } - if (!gCacheLoaded) { - await loadCache(); - } - let removed = false; - for (let key of keys) { - if (gCache.delete(key)) { - removed = true; - aiLog(`[AiClassifier] Removed cache entry '${key}'`, {debug: true}); - } - } - if (removed) { + const raw = await response.json(); + const entry = { ...adapter.parseResponse(raw), backend: snapshot.apiType, + model: typeof raw.model === 'string' ? raw.model : snapshot.model, version: 2, + messageHash: cacheKey?.split(':')[1] }; + if (cacheKey && epoch === generation) { + cache.set(cacheKey, entry); await saveCache(); } + return withPolicy(entry, threshold); } - -async function clearCache() { - if (!gCacheLoaded) { - await loadCache(); - } - if (gCache.size > 0) { - gCache.clear(); - await saveCache(); - aiLog(`[AiClassifier] Cleared cache`, {debug: true}); - } +export async function classifyText(text, criterion, cacheKey = null) { return (await classify(text, criterion, { cacheKey })).matched; } +export async function removeCacheEntries(keys = []) { + await init(); generation++; + for (const key of Array.isArray(keys) ? keys : [keys]) cache.delete(key); + await saveCache(); } - -async function getCacheSize() { - if (!gCacheLoaded) { - await loadCache(); - } - return gCache.size; +export async function clearCacheForMessage(id) { + await init(); + const identifier = await messageHash(id); + await removeCacheEntries([...cache].filter(([, entry]) => entry.messageHash === identifier).map(([key]) => key)); } - - -async function classifyText(text, criterion, cacheKey = null) { - if (!gCacheLoaded) { - await loadCache(); - } - const cached = getCachedResult(cacheKey); - if (cached !== null) { - return cached; - } - - const payload = buildPayload(text, criterion); - try { - await storage.local.set({ lastPayload: JSON.parse(payload) }); - } catch (e) { - aiLog('failed to save last payload', { level: 'warn' }, e); - } - - aiLog(`[AiClassifier] Sending classification request to ${gEndpoint}`, {debug: true}); - aiLog(`[AiClassifier] Classification request payload:`, { debug: true }, payload); - - try { - const response = await fetch(gEndpoint, { - method: "POST", - headers: { "Content-Type": "application/json", ...buildAuthHeaders() }, - body: payload, - }); - - if (!response.ok) { - aiLog(`HTTP status ${response.status}`, {level: 'warn'}); - return false; - } - - const result = await response.json(); - aiLog(`[AiClassifier] Received response:`, {debug: true}, result); - const parsed = parseMatch(result); - cacheEntry(cacheKey, parsed.matched, parsed.reason); - return parsed.matched; - } catch (e) { - aiLog(`HTTP request failed`, {level: 'error'}, e); - return false; - } -} - -async function init() { - await loadCache(); -} - -export { buildEndpointUrl, buildModelsUrl, normalizeEndpointBase, classifyText, setConfig, removeCacheEntries, clearCache, getReason, getCachedResult, buildCacheKey, getCacheSize, init }; +export async function clearCache() { await init(); generation++; cache.clear(); await saveCache(); } +export async function getCacheSize() { await init(); return cache.size; } diff --git a/modules/backends/chatCompletions.js b/modules/backends/chatCompletions.js new file mode 100644 index 0000000..38d9771 --- /dev/null +++ b/modules/backends/chatCompletions.js @@ -0,0 +1,81 @@ +const SYSTEM_PREFIX = `You are an email-classification assistant. +Read the email below and the classification criterion provided by the user. +`; + +const DEFAULT_CUSTOM_SYSTEM_PROMPT = "Determine whether the email satisfies the user's criterion."; + +const SYSTEM_SUFFIX = ` +Return ONLY a JSON object on a single line of the form: +{"match": true, "reason": ""} - if the email satisfies the criterion +{"match": false, "reason": ""} - otherwise + +Do not add any other keys, text, or formatting.`; + +function extractLastJsonObject(text) { + let last = null; + let start = -1; + let depth = 0; + let inString = false; + let escape = false; + + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (inString) { + if (escape) { + escape = false; + continue; + } + if (ch === "\\") { + escape = true; + continue; + } + if (ch === "\"") { + inString = false; + } + continue; + } + if (ch === "\"") { + inString = true; + continue; + } + if (ch === "{") { + if (depth === 0) { + start = i; + } + depth += 1; + continue; + } + if (ch === "}" && depth > 0) { + depth -= 1; + if (depth === 0 && start !== -1) { + last = text.slice(start, i + 1); + start = -1; + } + } + } + + return last; +} + +export const path = '/v1/chat/completions'; +export function buildRequest(text, criterion, config) { + const payload = { + messages: [ + { role: 'system', content: SYSTEM_PREFIX + (config.customSystemPrompt || DEFAULT_CUSTOM_SYSTEM_PROMPT) + SYSTEM_SUFFIX }, + { role: 'user', content: `**Email Contents**\n\`\`\`\n${text}\n\`\`\`\nClassification Criterion: ${criterion}` } + ], ...config.aiParams + }; + if (config.model) payload.model = config.model; + return payload; +} +export function parseResponse(result) { + const content = result?.choices?.[0]?.message?.content; + if (typeof content !== 'string') throw new Error('Chat response missing text content.'); + const candidate = extractLastJsonObject(content); + if (!candidate) throw new Error('No JSON object found in AI response.'); + const obj = JSON.parse(candidate); + const matched = Object.hasOwn(obj, 'match') ? obj.match : obj.matched; + if (typeof matched !== 'boolean') throw new Error('AI response missing valid match boolean.'); + const reason = obj.reason ?? obj.reasoning ?? obj.explaination; + return { matched, reason: typeof reason === 'string' ? reason : null, probability: null }; +} diff --git a/modules/backends/systemOne.js b/modules/backends/systemOne.js new file mode 100644 index 0000000..0de2472 --- /dev/null +++ b/modules/backends/systemOne.js @@ -0,0 +1,17 @@ +export const path = '/v1/systemone'; +export function buildRequest(text, criterion, config) { + const payload = { + state: text, + questions: { match: { type: 'noul', instructions: `Does this email satisfy the following criterion: ${criterion}` } } + }; + if (config.model) payload.model = config.model; + return payload; +} +export function parseResponse(result) { + const answer = result?.answers?.match; + if (answer?.type !== 'noul' || typeof answer.noul !== 'number' || + !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) { + throw new Error('SystemOne response must contain answers.match with type noul and a probability from 0 to 1.'); + } + return { matched: null, reason: null, probability: answer.noul }; +} diff --git a/modules/classificationConfig.js b/modules/classificationConfig.js new file mode 100644 index 0000000..3abe7cc --- /dev/null +++ b/modules/classificationConfig.js @@ -0,0 +1,32 @@ +import { DEFAULT_AI_PARAMS } from './defaultParams.js'; +export const CONFIG_DEFAULTS = { + apiType: 'chat-completions', endpoint: 'http://127.0.0.1:5000', model: '', + apiKey: '', openaiOrganization: '', openaiProject: '', customSystemPrompt: '', + decisionThreshold: 0.5, inputCharacterLimit: 3686, aiParams: DEFAULT_AI_PARAMS, + htmlToMarkdown: false, stripUrlParams: false, altTextImages: false, + collapseWhitespace: false, tokenReduction: false, debugLogging: false +}; +export function validateThreshold(value) { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error('Match threshold must be between 0% and 100%.'); + } + return value; +} +export function validateSettings(data) { + if (data.apiType !== undefined && !['chat-completions', 'systemone'].includes(data.apiType)) throw new Error('Unknown API type.'); + if (data.decisionThreshold !== undefined) validateThreshold(data.decisionThreshold); + if (data.inputCharacterLimit !== undefined && (!Number.isSafeInteger(data.inputCharacterLimit) || data.inputCharacterLimit < 1)) { + throw new Error('Input character limit must be a positive integer.'); + } + if (data.aiRules !== undefined) { + if (!Array.isArray(data.aiRules)) throw new Error('Rules must be an array.'); + for (const rule of data.aiRules) { + if (rule.decisionThreshold !== undefined) validateThreshold(rule.decisionThreshold); + } + } +} +export function migratedInputLimit(data) { + if (data.inputCharacterLimit !== undefined) return data.inputCharacterLimit; + const value = Math.floor(Number(data.aiParams?.max_tokens ?? 4096) * 0.9); + return Number.isSafeInteger(value) && value > 0 ? value : 3686; +} diff --git a/options/dataTransfer.js b/options/dataTransfer.js index 2c0b26e..ef2b237 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -1,7 +1,10 @@ "use strict"; +import { validateSettings, migratedInputLimit } from '../modules/classificationConfig.js'; +import { validCacheEntry } from '../modules/AiClassifier.js'; const storage = (globalThis.messenger ?? browser).storage; const KEY_GROUPS = { settings: [ + 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'tokenReduction', 'endpoint', 'model', 'apiKey', @@ -36,12 +39,15 @@ export async function exportData(categories) { URL.revokeObjectURL(url); } -export async function importData(file, categories) { +export async function importData(file, categories = Object.keys(KEY_GROUPS)) { const text = await file.text(); const parsed = JSON.parse(text); const data = {}; for (const key of collectKeys(categories)) { if (key in parsed) data[key] = parsed[key]; } + if (categories.includes('settings') && data.inputCharacterLimit === undefined) data.inputCharacterLimit = migratedInputLimit(data); + validateSettings(data); + if (data.aiCache !== undefined) data.aiCache = Object.fromEntries(Object.entries(data.aiCache || {}).filter(([key, entry]) => validCacheEntry(key, entry))); await storage.local.set(data); } diff --git a/options/options.html b/options/options.html index cd3d581..8091df5 100644 --- a/options/options.html +++ b/options/options.html @@ -64,11 +64,25 @@
    + +

    Settings

    +
    + +
    +
    +
    + + +

    Match when probability is at least this value. Rules may override it. Evaluate thresholds on your own email examples.

    +
    @@ -186,7 +200,12 @@
    - + + +

    Applied with aggressive reduction. Characters are not tokens; leave room for instructions within your model's context limit.

    +
    +
    +
    diff --git a/options/options.js b/options/options.js index 2a8d3c4..d1f71a2 100644 --- a/options/options.js +++ b/options/options.js @@ -5,7 +5,9 @@ document.addEventListener('DOMContentLoaded', async () => { const dataTransfer = await import(browser.runtime.getURL('options/dataTransfer.js')); const { detectSystemTheme } = await import(browser.runtime.getURL('modules/themeUtils.js')); const { DEFAULT_AI_PARAMS } = await import(browser.runtime.getURL('modules/defaultParams.js')); + const { validateSettings, migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); const defaults = await storage.local.get([ + 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'endpoint', 'customSystemPrompt', 'model', @@ -86,6 +88,23 @@ document.addEventListener('DOMContentLoaded', async () => { markDirty(); await applyTheme(themeSelect.value); }); + const apiTypeInput = document.getElementById('api-type'); + apiTypeInput.value = defaults.apiType || 'chat-completions'; + const thresholdInput = document.getElementById('decision-threshold'); + thresholdInput.value = (defaults.decisionThreshold ?? 0.5) * 100; + const inputLimit = document.getElementById('input-character-limit'); + inputLimit.value = migratedInputLimit(defaults); + const settingsError = document.getElementById('settings-error'); + function updateApiControls() { + const decision = apiTypeInput.value === 'systemone'; + document.querySelectorAll('.systemone-only').forEach(el => el.classList.toggle('is-hidden', !decision)); + for (const id of ['system-instructions', 'openai-organization', 'openai-project', ...Object.keys(DEFAULT_AI_PARAMS)]) { + document.getElementById(id)?.closest('.field')?.classList.toggle('is-hidden', decision); + } + document.getElementById('reset-system').classList.toggle('is-hidden', decision); + updateEndpointPreview(); + } + apiTypeInput.addEventListener('change', updateApiControls); const endpointInput = document.getElementById('endpoint'); const endpointPreview = document.getElementById('endpoint-preview'); const fallbackEndpoint = 'http://127.0.0.1:5000'; @@ -94,13 +113,13 @@ document.addEventListener('DOMContentLoaded', async () => { endpointInput.value = endpointBase; function updateEndpointPreview() { - const resolved = AiClassifier.buildEndpointUrl(endpointInput.value); + const resolved = AiClassifier.buildEndpointUrl(endpointInput.value, apiTypeInput.value); endpointPreview.textContent = resolved ? `Resolved endpoint: ${resolved}` : 'Resolved endpoint: (invalid)'; } endpointInput.addEventListener('input', updateEndpointPreview); - updateEndpointPreview(); + updateApiControls(); const modelSelect = document.getElementById('model-select'); const refreshModelsBtn = document.getElementById('refresh-models'); @@ -148,11 +167,11 @@ document.addEventListener('DOMContentLoaded', async () => { headers.Authorization = `Bearer ${apiKey}`; } const organization = openaiOrgInput?.value.trim(); - if (organization) { + if (organization && apiTypeInput.value === 'chat-completions') { headers["OpenAI-Organization"] = organization; } const project = openaiProjectInput?.value.trim(); - if (project) { + if (project && apiTypeInput.value === 'chat-completions') { headers["OpenAI-Project"] = project; } return headers; @@ -467,7 +486,7 @@ document.addEventListener('DOMContentLoaded', async () => { const perDayEl = document.getElementById('per-day'); let timingLogged = false; ruleCountEl.textContent = (defaults.aiRules || []).length; - cacheCountEl.textContent = defaults.aiCache ? Object.keys(defaults.aiCache).length : 0; + cacheCountEl.textContent = '…'; function createActionRow(action = {type: 'tag'}) { const row = document.createElement('div'); @@ -861,14 +880,25 @@ document.addEventListener('DOMContentLoaded', async () => { body.appendChild(folderBox); article.appendChild(header); + const thresholdField = document.createElement('label'); + thresholdField.className = 'field systemone-only'; + thresholdField.classList.toggle('is-hidden', apiTypeInput.value !== 'systemone'); + thresholdField.textContent = 'Match probability threshold (%) — blank uses default'; + const override = document.createElement('input'); + override.className = 'input decision-threshold'; + override.type = 'number'; + override.min = '0'; override.max = '100'; override.step = 'any'; + override.value = rule.decisionThreshold === undefined ? '' : rule.decisionThreshold * 100; + thresholdField.appendChild(override); + body.appendChild(thresholdField); article.appendChild(body); rulesContainer.appendChild(article); } } - addRuleBtn.addEventListener('click', () => { - const data = [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { + function readRules() { + return [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { const criterion = ruleEl.querySelector('.criterion').value; const actions = [...ruleEl.querySelectorAll('.action-row')].map(row => { const type = row.querySelector('select').value; @@ -890,8 +920,11 @@ document.addEventListener('DOMContentLoaded', async () => { if (type === 'flag') { return { type, flagged: row.querySelector('.flag-select').value === 'true' }; } - if (type === 'delete' || type === 'archive') { - return { type }; + if (type === 'forward') { + return { type, address: row.querySelector('.forward-input').value.trim() }; + } + if (type === 'reply') { + return { type, replyType: row.querySelector('.reply-select').value }; } return { type }; }); @@ -903,6 +936,8 @@ document.addEventListener('DOMContentLoaded', async () => { const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); const rule = { criterion, actions, enabled }; + const override = ruleEl.querySelector('.decision-threshold').value; + if (override.trim() !== '') rule.decisionThreshold = Number(override) / 100; if (unreadOnly) rule.unreadOnly = true; if (stopProcessing) rule.stopProcessing = true; if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; @@ -911,6 +946,9 @@ document.addEventListener('DOMContentLoaded', async () => { if (folders.length) rule.folders = folders; return rule; }); + } + addRuleBtn.addEventListener('click', () => { + const data = readRules(); data.push({ criterion: '', actions: [], unreadOnly: false, stopProcessing: false, enabled: true, accounts: [], folders: [] }); renderRules(data); }); @@ -927,6 +965,7 @@ document.addEventListener('DOMContentLoaded', async () => { if (r.moveTo) actions.push({ type: 'move', folder: r.moveTo }); if (r.copyTarget || r.copyTo) actions.push({ type: 'copy', copyTarget: r.copyTarget || r.copyTo }); const rule = { criterion: r.criterion, actions }; + if (r.decisionThreshold !== undefined) rule.decisionThreshold = r.decisionThreshold; if (r.stopProcessing) rule.stopProcessing = true; if (r.unreadOnly) rule.unreadOnly = true; if (typeof r.minAgeDays === 'number') rule.minAgeDays = r.minAgeDays; @@ -996,8 +1035,8 @@ document.addEventListener('DOMContentLoaded', async () => { } try { - const { aiCache } = await storage.local.get('aiCache'); - cacheCountEl.textContent = aiCache ? Object.keys(aiCache).length : 0; + const { count } = await browser.runtime.sendMessage({ type: 'sortana:getCacheSize' }); + cacheCountEl.textContent = count; } catch { cacheCountEl.textContent = '?'; } @@ -1044,14 +1083,22 @@ document.addEventListener('DOMContentLoaded', async () => { document.getElementById('import-data').addEventListener('click', () => importInput.click()); importInput.addEventListener('change', async () => { if (importInput.files.length) { - await dataTransfer.importData(importInput.files[0], selectedCategories()); - location.reload(); + try { + await dataTransfer.importData(importInput.files[0], selectedCategories()); + location.reload(); + } catch (error) { + window.alert(`Import failed: ${error.message}`); + } } }); initialized = true; document.getElementById('save').addEventListener('click', async () => { + settingsError.textContent = ''; + const apiType = apiTypeInput.value; + const decisionThreshold = thresholdInput.value.trim() === '' ? NaN : Number(thresholdInput.value) / 100; + const inputCharacterLimit = Number(inputLimit.value); const endpoint = endpointInput.value.trim(); const model = modelSelect?.value || ''; const apiKey = apiKeyInput?.value.trim() || ''; @@ -1068,63 +1115,25 @@ document.addEventListener('DOMContentLoaded', async () => { } const debugLogging = debugToggle.checked; const htmlToMarkdown = htmlToggle.checked; - const rules = [...rulesContainer.querySelectorAll('.rule')].map(ruleEl => { - const criterion = ruleEl.querySelector('.criterion').value; - const actions = [...ruleEl.querySelectorAll('.action-row')].map(row => { - const type = row.querySelector('select').value; - if (type === 'tag') { - return { type, tagKey: row.querySelector('.tag-select').value }; - } - if (type === 'move') { - return { type, folder: row.querySelector('.folder-select').value }; - } - if (type === 'copy') { - return { type, copyTarget: row.querySelector('.folder-select').value }; - } - if (type === 'junk') { - return { type, junk: row.querySelector('.junk-select').value === 'true' }; - } - if (type === 'read') { - return { type, read: row.querySelector('.read-select').value === 'true' }; - } - if (type === 'flag') { - return { type, flagged: row.querySelector('.flag-select').value === 'true' }; - } - if (type === 'forward') { - return { type, address: row.querySelector('.forward-input').value.trim() }; - } - if (type === 'reply') { - return { type, replyType: row.querySelector('.reply-select').value }; - } - return { type }; - }); - const stopProcessing = ruleEl.querySelector('.stop-processing')?.checked; - const unreadOnly = ruleEl.querySelector('.unread-only')?.checked; - const enabled = ruleEl.dataset.enabled !== 'false'; - const minAgeDays = parseFloat(ruleEl.querySelector('.min-age')?.value); - const maxAgeDays = parseFloat(ruleEl.querySelector('.max-age')?.value); - const accounts = [...(ruleEl.querySelector('.account-select')?.selectedOptions || [])].map(o => o.value); - const folders = [...(ruleEl.querySelector('.folder-filter-select')?.selectedOptions || [])].map(o => o.value); - const rule = { criterion, actions, enabled }; - if (unreadOnly) rule.unreadOnly = true; - if (stopProcessing) rule.stopProcessing = true; - if (!isNaN(minAgeDays)) rule.minAgeDays = minAgeDays; - if (!isNaN(maxAgeDays)) rule.maxAgeDays = maxAgeDays; - if (accounts.length) rule.accounts = accounts; - if (folders.length) rule.folders = folders; - return rule; - }).filter(r => r.criterion); + const invalidThreshold = [...document.querySelectorAll('.decision-threshold'), thresholdInput, inputLimit].find(el => !el.checkValidity()); + if (invalidThreshold) { + settingsError.textContent = 'Enter thresholds from 0% to 100% and a positive whole-number character limit.'; + return; + } + const rules = readRules().filter(r => r.criterion); const stripUrlParams = stripUrlToggle.checked; const altTextImages = altTextToggle.checked; const collapseWhitespace = collapseWhitespaceToggle.checked; const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - await storage.local.set({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + try { validateSettings({ apiType, decisionThreshold, inputCharacterLimit, aiRules: rules }); } + catch (error) { settingsError.textContent = error.message; return; } + await storage.local.set({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); await storage.local.remove(['templateName', 'customTemplate']); await applyTheme(theme); try { - await AiClassifier.setConfig({ endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); diff --git a/tests/cacheRestore.test.mjs b/tests/cacheRestore.test.mjs new file mode 100644 index 0000000..f5ca74d --- /dev/null +++ b/tests/cacheRestore.test.mjs @@ -0,0 +1,84 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import { readFile } from 'node:fs/promises'; +import { CONFIG_DEFAULTS } from '../modules/classificationConfig.js'; + +const persisted = {}; +const listeners = []; +const storage = { + local: { + async get(keys) { + return Object.fromEntries((Array.isArray(keys) ? keys : [keys]) + .filter(key => key in persisted).map(key => [key, structuredClone(persisted[key])])); + }, + async set(values) { + const changes = {}; + for (const [key, value] of Object.entries(values)) { + changes[key] = { oldValue: persisted[key], newValue: structuredClone(value) }; + persisted[key] = structuredClone(value); + } + for (const listener of listeners) await listener(changes, 'local'); + }, + async remove(keys) { + for (const key of Array.isArray(keys) ? keys : [keys]) delete persisted[key]; + } + }, + onChanged: { addListener(listener) { listeners.push(listener); } } +}; +const messenger = { storage, messages: { async get(id) { + // Numeric Thunderbird IDs may change after reinstall; RFC Message-ID does not. + return { subject: 'Invoice', headerMessageId: 'persistent-message@example.com' }; +} } }; +globalThis.messenger = messenger; +const classifier = await import('../modules/AiClassifier.js'); +const transfer = await import('../options/dataTransfer.js'); +const source = await readFile(new URL('../background.js', import.meta.url), 'utf8'); +let onMessage; +const context = vm.createContext({ + storage, messenger, structuredClone, AiClassifier: classifier, aiRules: [], + logger: { aiLog() {} }, logGetTiming: false, + htmlToMarkdown: false, stripUrlParams: false, altTextImages: false, + collapseWhitespace: false, tokenReduction: false, showDebugTab: false, + browser: { runtime: { onMessage: { addListener(listener) { onMessage = listener; } } } } +}); +// Exercise the actual background storage and Details handlers, not a manual cache reload. +const normalizeStart = source.indexOf('function normalizeRules('); +vm.runInContext(source.slice(normalizeStart, source.indexOf('function iconPaths(', normalizeStart)), context); +const changesStart = source.indexOf('storage.onChanged.addListener('); +vm.runInContext(source.slice(changesStart, source.indexOf('\n\n if (browser.theme', changesStart)), context); +const messageStart = source.indexOf('browser.runtime.onMessage.addListener('); +vm.runInContext(source.slice(messageStart, source.indexOf('\n\n // Automatically classify', messageStart)), context); + +test('export/reinstall/import restores Details immediately with changed Thunderbird IDs', async () => { + const settings = { ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 }; + const rules = [{ criterion: 'invoice', decisionThreshold: 0.8, actions: [] }]; + await storage.local.set({ ...settings, aiRules: rules }); + globalThis.fetch = async () => ({ ok: true, json: async () => ({ answers: { match: { type: 'noul', noul: 0.9 } } }) }); + const key = await classifier.buildCacheKey(1, 'invoice'); + await classifier.classify('Invoice attached', 'invoice', { cacheKey: key, threshold: 0.8 }); + let exported; + const originalDocument = globalThis.document; + const createURL = URL.createObjectURL; + const revokeURL = URL.revokeObjectURL; + URL.createObjectURL = blob => { exported = blob; return 'blob:export'; }; + URL.revokeObjectURL = () => {}; + globalThis.document = { body: { appendChild() {} }, createElement() { return { click() {}, remove() {} }; } }; + try { await transfer.exportData(['settings', 'rules', 'cache']); } + finally { globalThis.document = originalDocument; URL.createObjectURL = createURL; URL.revokeObjectURL = revokeURL; } + await classifier.clearCache(); + for (const key of Object.keys(persisted)) delete persisted[key]; + classifier.setConfig(CONFIG_DEFAULTS); + context.aiRules = []; + assert.equal((await onMessage({ type: 'sortana:getDetails', id: 99 })).results.length, 0); + await transfer.importData(exported, ['settings', 'rules', 'cache']); + assert.equal((await onMessage({ type: 'sortana:getCacheSize' })).count, 1); + const details = await onMessage({ type: 'sortana:getDetails', id: 99 }); + assert.equal(details.results.length, 1); + assert.equal(details.results[0].criterion, 'invoice'); + assert.equal(details.results[0].probability, 0.9); + assert.equal(details.results[0].threshold, 0.8); + assert.equal(details.results[0].matched, true); + globalThis.fetch = () => { throw new Error('Restored cache must avoid inference'); }; + assert.equal((await classifier.classify('Invoice attached', 'invoice', { cacheKey: await classifier.buildCacheKey(99, 'invoice') })).matched, true); +}); diff --git a/tests/classification.test.mjs b/tests/classification.test.mjs new file mode 100644 index 0000000..daf6ea6 --- /dev/null +++ b/tests/classification.test.mjs @@ -0,0 +1,167 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as chat from '../modules/backends/chatCompletions.js'; +import * as decision from '../modules/backends/systemOne.js'; +import { CONFIG_DEFAULTS, migratedInputLimit, validateSettings } from '../modules/classificationConfig.js'; + +const data = { aiCache: { legacy: { matched: true, reason: 'old' } }, aiReasonCache: { legacy: 'old' } }; +globalThis.messenger = { storage: { local: { + async get(keys) { return Object.fromEntries((Array.isArray(keys) ? keys : [keys]).map(key => [key, data[key]])); }, + async set(values) { Object.assign(data, structuredClone(values)); }, + async remove(keys) { for (const key of Array.isArray(keys) ? keys : [keys]) delete data[key]; } +} }, messages: { async get(id) { return { headerMessageId: `message-${id}` }; } } }; +const classifier = await import('../modules/AiClassifier.js'); +const transfer = await import('../options/dataTransfer.js'); +const answer = probability => ({ model: 'served-model', answers: { match: { type: 'noul', noul: probability } } }); +let calls = []; +function serve(body, status = 200) { + globalThis.fetch = async (url, request) => { calls.push({ url, ...request }); return { ok: status === 200, status, json: async () => body }; }; +} + +test('adapters preserve chat prompts and isolate SystemOne payloads', () => { + const config = { ...CONFIG_DEFAULTS, model: 'chosen', customSystemPrompt: 'Custom guidance' }; + const payload = chat.buildRequest('email', 'invoice', config); + assert.match(payload.messages[0].content, /Custom guidance/); + assert.match(payload.messages[1].content, /Classification Criterion: invoice/); + assert.equal(payload.max_tokens, 4096); + assert.deepEqual(decision.buildRequest('email', 'invoice', config), { + state: 'email', model: 'chosen', questions: { match: { type: 'noul', instructions: 'Does this email satisfy the following criterion: invoice' } } + }); + assert.equal('model' in decision.buildRequest('email', 'invoice', CONFIG_DEFAULTS), false); + assert.equal('model' in chat.buildRequest('email', 'invoice', CONFIG_DEFAULTS), false); +}); + +test('parsers validate evidence including zero, one, and malformed answers', () => { + for (const probability of [0, 0.5, 1]) assert.equal(decision.parseResponse(answer(probability)).probability, probability); + for (const probability of [null, '0.5', -0.1, 1.1, Infinity, NaN]) assert.throws(() => decision.parseResponse(answer(probability))); + for (const raw of [null, {}, { answers: { other: { type: 'noul', noul: 0.5 } } }, { answers: { match: { type: 'choice', noul: 0.5 } } }]) assert.throws(() => decision.parseResponse(raw)); + assert.deepEqual(chat.parseResponse({ choices: [{ message: { content: 'text {"match":false} then {"matched":true,"reason":"brace } inside"}' } }] }), { matched: true, reason: 'brace } inside', probability: null }); + for (const content of ['', '{broken}', '{"match":"true"}']) assert.throws(() => chat.parseResponse({ choices: [{ message: { content } }] })); + assert.throws(() => chat.parseResponse(null)); +}); + +test('URLs preserve proxy prefixes and normalize API suffixes', () => { + for (const suffix of ['', '/', '/v1', '/v1/', '/v1/models', '/v1/chat/completions/', '/v1/systemone']) { + assert.equal(classifier.buildEndpointUrl(`http://localhost:8080/proxy${suffix}`, 'systemone'), 'http://localhost:8080/proxy/v1/systemone'); + assert.equal(classifier.buildModelsUrl(`http://localhost:8080/proxy${suffix}`), 'http://localhost:8080/proxy/v1/models'); + } + assert.equal(classifier.buildEndpointUrl('example.com'), 'https://example.com/v1/chat/completions'); + assert.equal(classifier.buildEndpointUrl(''), ''); +}); + +test('settings validation and independent character-limit migration', () => { + assert.equal(migratedInputLimit({}), 3686); + assert.equal(migratedInputLimit({ aiParams: { max_tokens: 1000 } }), 900); + assert.equal(migratedInputLimit({ inputCharacterLimit: 500, aiParams: { max_tokens: 1000 } }), 500); + validateSettings({ decisionThreshold: 0, aiRules: [{ decisionThreshold: 1 }] }); + for (const value of [-1, 2, null, '0.5']) assert.throws(() => validateSettings({ aiRules: [{ decisionThreshold: value }] })); + for (const value of [0, 1.2, NaN]) assert.throws(() => validateSettings({ inputCharacterLimit: value })); +}); + +test('cache migration, inference reuse, fingerprints, clearing and import', async () => { + await classifier.init(); + assert.equal(await classifier.getCacheSize(), 0); + assert.equal(data.aiReasonCache, undefined); + assert.deepEqual(data.aiCache, {}); + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://localhost:8080', apiKey: 'secret', openaiOrganization: 'org', openaiProject: 'project' }); + const snapshot = classifier.getConfig(); + const key = await classifier.buildCacheKey(1, 'invoice', snapshot); + serve(answer(0.75)); calls = []; + const first = await classifier.classify('email', 'invoice', { snapshot, cacheKey: key, threshold: 0.75 }); + assert.equal(first.matched, true); + assert.equal(first.model, 'served-model'); + assert.deepEqual(calls[0].headers, { 'Content-Type': 'application/json', Authorization: 'Bearer secret' }); + const second = await classifier.classify('email', 'invoice', { snapshot, cacheKey: key, threshold: 0.8 }); + assert.equal(second.matched, false); + assert.equal(second.probability, 0.75); + assert.equal(calls.length, 1); + classifier.replaceCache(data.aiCache); + assert.equal(classifier.getCachedDetails(key, 0.75).matched, true); + assert.equal(await classifier.buildCacheKey(1, 'invoice', { ...snapshot, decisionThreshold: 0.9, apiKey: 'new secret' }), key); + for (const change of [{ apiType: 'chat-completions' }, { endpoint: 'http://other' }, { model: 'other' }, { tokenReduction: true }, { inputCharacterLimit: 500 }]) { + assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...snapshot, ...change }), key); + } + const chatSnapshot = { ...snapshot, apiType: 'chat-completions' }; + const chatKey = await classifier.buildCacheKey(1, 'invoice', chatSnapshot); + assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...chatSnapshot, customSystemPrompt: 'other' }), chatKey); + assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...chatSnapshot, aiParams: { ...snapshot.aiParams, temperature: 1 } }), chatKey); + serve({ choices: [{ message: { content: '{"match":false,"reason":"not invoice"}' } }] }); + await classifier.classify('email', 'invoice', { cacheKey: chatKey, snapshot: chatSnapshot }); + assert.equal(calls.at(-1).headers['OpenAI-Organization'], 'org'); + assert.equal(await classifier.getCacheSize(), 2); + const exported = structuredClone(data.aiCache); + await classifier.clearCacheForMessage(1); + assert.equal(await classifier.getCacheSize(), 0); + await transfer.importData({ text: async () => JSON.stringify({ aiCache: { ...exported, legacy: true } }) }, ['cache']); + classifier.replaceCache(data.aiCache); // background storage listener consumes imported replacement + assert.equal(await classifier.getCacheSize(), 2); + await assert.rejects(transfer.importData({ text: async () => JSON.stringify({ decisionThreshold: 2 }) }, ['settings'])); + await assert.rejects(transfer.importData({ text: async () => JSON.stringify({ aiRules: [{ decisionThreshold: -1 }] }) }, ['rules'])); + await classifier.clearCache(); +}); + +test('failures remain uncached and snapshots survive configuration changes', async () => { + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone' }); + const snapshot = classifier.getConfig(); + const key = await classifier.buildCacheKey(2, 'invoice', snapshot); + for (const raw of [null, answer('0.9'), {}]) { + serve(raw); + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot })); + assert.equal(classifier.getCachedDetails(key), null); + } + serve({}, 503); + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /503/); + globalThis.fetch = async () => { throw new Error('network'); }; + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /network/); + globalThis.fetch = async () => ({ ok: true, json: async () => { throw new SyntaxError('bad JSON'); } }); + await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /bad JSON/); + classifier.setConfig({ apiType: 'chat-completions', endpoint: 'http://changed' }); + serve(answer(1)); calls = []; + const result = await classifier.classify('email', 'invoice', { cacheKey: key, snapshot }); + assert.equal(result.backend, 'systemone'); + assert.equal(calls[0].url, 'http://127.0.0.1:5000/v1/systemone'); + assert.equal('Authorization' in calls[0].headers, false); + await classifier.clearCache(); +}); + +test('clearing or replacing cache during inference prevents stale repopulation', async () => { + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone' }); + const snapshot = classifier.getConfig(); + const key = await classifier.buildCacheKey(3, 'invoice', snapshot); + for (const replace of [() => classifier.clearCache(), () => classifier.replaceCache({})]) { + let release, started; + const ready = new Promise(resolve => { started = resolve; }); + globalThis.fetch = async () => { + started(); + await new Promise(resolve => { release = resolve; }); + return { ok: true, json: async () => answer(0.9) }; + }; + const pending = classifier.classify('email', 'invoice', { cacheKey: key, snapshot }); + await ready; + await replace(); + release(); + assert.equal((await pending).matched, true); + assert.equal(classifier.getCachedDetails(key), null); + } +}); + +test('settings and rule imports retain thresholds and reject invalid caches', async () => { + const settings = { apiType: 'systemone', decisionThreshold: 0.75, inputCharacterLimit: 5000, tokenReduction: true }; + const rules = [{ criterion: 'invoice', decisionThreshold: 0, actions: [{ type: 'flag', flagged: true }] }, { criterion: 'urgent', actions: [] }]; + await transfer.importData({ text: async () => JSON.stringify({ ...settings, aiRules: rules }) }, ['settings', 'rules']); + for (const [key, value] of Object.entries(settings)) assert.equal(data[key], value); + assert.deepEqual(data.aiRules, rules); + const key = await classifier.buildCacheKey(4, 'invoice'); + assert.equal(classifier.validCacheEntry(key, { version: 2, backend: 'systemone', probability: '0.8', reason: null, messageHash: key.split(':')[1], model: '' }), false); +}); + +test('SystemOne 404 reports endpoint, server message, and compatibility guidance', async () => { + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://seinei.wageshouse:5003' }); + serve({ error: { message: 'File Not Found' } }, 404); + await assert.rejects(classifier.classify('synthetic email', 'invoice'), error => { + assert.match(error.message, /HTTP 404 at http:\/\/seinei\.wageshouse:5003\/v1\/systemone/); + assert.match(error.message, /Server response: File Not Found/); + assert.match(error.message, /running server build supports/); + return true; + }); +}); diff --git a/tests/ruleExecution.test.mjs b/tests/ruleExecution.test.mjs new file mode 100644 index 0000000..153a050 --- /dev/null +++ b/tests/ruleExecution.test.mjs @@ -0,0 +1,80 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import { readFile } from 'node:fs/promises'; + +// Run the actual background rule/queue functions with Thunderbird services mocked. +const source = await readFile(new URL('../background.js', import.meta.url), 'utf8'); +const functions = source.slice(source.indexOf('async function processMessage('), source.indexOf('async function clearCacheForMessages(')); +function runner({ fail = false, changeSettings = false } = {}) { + const actions = [], evaluated = [], errors = []; + const rules = [ + { criterion: 'first', decisionThreshold: 0.8, actions: [{ type: 'flag', flagged: true }], stopProcessing: true }, + { criterion: 'second', actions: [{ type: 'read', read: true }] } + ]; + let currentConfig = { apiType: 'systemone', inputCharacterLimit: 5, decisionThreshold: 0.5 }; + const context = vm.createContext({ + structuredClone, Date, Error, + aiRules: rules, queue: Promise.resolve(), queuedCount: 0, processing: false, + currentStart: 0, tokenReduction: true, showDebugTab: false, timingStats: {}, + updateActionIcon() {}, updateTimingStats() {}, showTransientIcon() {}, + buildEmailText() { return 'email contents'; }, ICONS: { circle: 'circle' }, + logger: { aiLog() {} }, recordError(...args) { errors.push(args); }, + ERROR_NOTIFICATION_ID: 'error', + storage: { local: { async set() {} } }, + messenger: { + messages: { + async getFull() { return {}; }, + async get() { + if (changeSettings) { + rules[0].decisionThreshold = 0.1; + currentConfig = { apiType: 'chat-completions', decisionThreshold: 0.1 }; + } + return { tags: [], folder: { accountId: 'account' } }; + }, + async update(id, action) { actions.push({ id, ...action }); } + }, identities: { async list() { return []; } } + }, + browser: { runtime: { getURL: x => x }, notifications: { async create() {} } }, + AiClassifier: { + getConfig() { return structuredClone(currentConfig); }, + async buildCacheKey(id, criterion) { return `${id}:${criterion}`; }, + async classify(text, criterion, options) { + evaluated.push({ text, criterion, ...options }); + if (fail && options.cacheKey.startsWith('1:')) throw new Error('inference failed'); + return { matched: true }; + } + } + }); + vm.runInContext(functions, context); + return { context, actions, evaluated, errors }; +} + +test('failure stops remaining rules for one email, next queued email still acts', async () => { + const { context, actions, evaluated, errors } = runner({ fail: true }); + await vm.runInContext('applyAiRules([1, 2])', context); + assert.deepEqual(evaluated.map(x => x.cacheKey), ['1:first', '2:first']); + assert.deepEqual(actions, [{ id: 2, flagged: true }]); + assert.equal(errors.length, 1); + assert.equal(context.processing, false); + assert.equal(context.queuedCount, 0); +}); + +test('settings and rules remain fixed during an email; reduction uses characters', async () => { + const { context, evaluated, actions } = runner({ changeSettings: true }); + await vm.runInContext('applyAiRules([1])', context); + assert.equal(evaluated[0].text, 'email'); + assert.equal(evaluated[0].threshold, 0.8); + assert.equal(evaluated[0].snapshot.apiType, 'systemone'); + assert.equal(evaluated.length, 1); // stop after first match + assert.equal(actions.length, 1); +}); + +test('inherited thresholds and eligibility preserve ordered rule processing', async () => { + const { context, evaluated, actions } = runner(); + context.aiRules[0].enabled = false; + await vm.runInContext('applyAiRules([1])', context); + assert.equal(evaluated[0].criterion, 'second'); + assert.equal(evaluated[0].threshold, 0.5); + assert.deepEqual(actions, [{ id: 1, read: true }]); +}); From 7edae937b3fe7fcfd2be4ad47c48330469870ec8 Mon Sep 17 00:00:00 2001 From: wagesj45 Date: Thu, 8 Oct 2026 09:42:26 -0500 Subject: [PATCH 124/124] Add configurable SystemOne batching and release version 4.1.0 --- AGENTS.md | 27 +++- README.md | 20 ++- background.js | 9 +- manifest.json | 2 +- modules/AiClassifier.js | 76 +++++++++-- modules/backends/systemOne.js | 19 ++- modules/classificationConfig.js | 7 + options/dataTransfer.js | 1 + options/options.html | 9 ++ options/options.js | 27 +++- tests/batching.test.mjs | 219 ++++++++++++++++++++++++++++++++ tests/cacheRestore.test.mjs | 4 +- tests/ruleExecution.test.mjs | 11 +- 13 files changed, 400 insertions(+), 31 deletions(-) create mode 100644 tests/batching.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 673e422..6a3d1e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,8 +48,10 @@ sent only for Chat Completions. Chat Completions sends system/user messages and extracts the last JSON object from `choices[0].message.content`, requiring a `match` or `matched` boolean. SystemOne -sends prepared email text as `state` and a single `questions.match` of type `noul`. -It requires `answers.match.type === "noul"` and a finite numeric `noul` in `[0, 1]`. +sends prepared email text as `state` with `noul` questions identified by stable +`rule_` IDs. The single-question API still uses `match` for backward +compatibility and cache fingerprinting. Every requested answer must have type +`noul` and a finite numeric probability in `[0, 1]`; response order is irrelevant. No chat generation parameters or system prompt are sent to SystemOne. The global `decisionThreshold` defaults to 0.5; optional rule-level @@ -67,6 +69,27 @@ error, and continues the next queued email. Snapshot settings and rules per emai `inputCharacterLimit` replaces coupling input truncation to output `max_tokens`; it applies with aggressive reduction and migrates from the old effective limit. +### SystemOne batching + +`systemOneProcessAll` defaults to true; `systemOneBatchSize` defaults to 8 and must +be a positive safe integer. Only SystemOne shows these controls. The numeric input +is disabled while Process all is enabled, retaining its value. Include both in +settings import/export, startup loading, live updates, and per-email snapshots. + +`classifyRules()` is a lazy ordered async iterator over precomputed candidate keys. +Chat requests stay sequential. SystemOne batches the next N unique uncached keys +(or all), sends state once, validates the whole response, and persists one batch +of evidence before yielding ordered results. Per-rule thresholds remain independent. +Do not change existing cache keys or include batch settings in their fingerprint. +Prepare eligibility and keys before actions, using the existing initial mail metadata. + +Stop-after-match ends action iteration and prevents further requests; already +computed later answers remain cached. A malformed answer fails its entire batch: +no partial cache writes or new actions. Keep cache-generation guards for imports +and clearing during inference. No automatic splitting or retries. Tests in +`tests/batching.test.mjs` cover batching, cache reuse, failure atomicity, and actual +rule-runner stop behavior with the real classifier. + ## Documentation Additional documentation exists outside this repository. diff --git a/README.md b/README.md index b7fb0f5..999d826 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ llama-server -hf ggml-org/Laya-GGUF for a single loaded model. Router deployments require the appropriate model ID. 2. Set the default match probability threshold (initially **50%**). Each rule has an optional percentage override; leave it blank to inherit the default. -3. Apply rules normally. One `noul` question is sent per evaluated rule. A rule +3. Apply rules normally. Each uncached criterion becomes a `noul` question. A rule matches when the unrounded probability is **greater than or equal to** its effective threshold. For example, 80% matches an 80% threshold. 4. Open Details to see a probability bar with the percentage inside. Hover over @@ -83,6 +83,24 @@ llama-server -hf ggml-org/Laya-GGUF not automatically rerun actions. Decision models provide no textual explanation; no second model is called to invent one. Use Chat Completions for explanations. +**SystemOne batching:** **Process all** is enabled by default and sends all eligible, +uncached criteria for an email in one request, with the email state included once. +Turn it off to set **Rules per request** (initially 8); use 1 for sequential +requests. The number is retained while disabled. These controls apply only to +SystemOne; Chat Completions continues processing one rule per request. + +Cached rules consume no batch capacity, and identical criteria share a question +while retaining their individual thresholds. Actions execute in rule order and +honor stop-after-match. A batch can evaluate later rules whose actions are then +skipped; their probabilities remain available in Details. No further batches are +requested after stopping. Changing batch settings preserves existing cached results. + +If any requested answer is missing or invalid, the entire batch fails without +caching its new answers or executing their actions. Previously completed actions +remain applied. Sortana does not automatically retry or split rejected batches; +if the server rejects a large request, disable Process all and reduce Rules per +request. Limits and performance depend on the server and model. + Probabilities are model estimates, not guaranteed accuracy. Evaluate thresholds against representative labeled emails, including long messages and attachments. SystemOne uses the existing text extraction and attachment summaries; it does not diff --git a/background.js b/background.js index 1b9f554..602fe1f 100644 --- a/background.js +++ b/background.js @@ -347,7 +347,8 @@ async function processMessage(id) { identityId = null; } - for (const rule of rules) { + const candidates = []; + for (const [index, rule] of rules.entries()) { if (rule.enabled === false) { continue; } @@ -375,7 +376,9 @@ async function processMessage(id) { } } const cacheKey = await AiClassifier.buildCacheKey(id, rule.criterion, snapshot); - const { matched } = await AiClassifier.classify(text, rule.criterion, { cacheKey, snapshot, threshold: rule.decisionThreshold ?? snapshot.decisionThreshold }); + candidates.push({ id: `rule_${index}`, rule, cacheKey }); + } + for await (const { rule, result: { matched } } of AiClassifier.classifyRules(text, candidates, snapshot)) { if (matched) { for (const act of (rule.actions || [])) { if (act.type === 'tag' && act.tagKey) { @@ -476,7 +479,7 @@ async function clearCacheForMessages(idsInput) { } try { - const store = await storage.local.get(["apiType", "decisionThreshold", "inputCharacterLimit", "endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); + const store = await storage.local.get(["systemOneProcessAll", "systemOneBatchSize", "apiType", "decisionThreshold", "inputCharacterLimit", "endpoint", "model", "apiKey", "openaiOrganization", "openaiProject", "customSystemPrompt", "aiParams", "debugLogging", "htmlToMarkdown", "stripUrlParams", "altTextImages", "collapseWhitespace", "tokenReduction", "aiRules", "theme", "showDebugTab"]); logger.setDebug(store.debugLogging); const { migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); if (store.inputCharacterLimit === undefined) { diff --git a/manifest.json b/manifest.json index e9a3088..b304bc0 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Sortana", - "version": "4.0.0", + "version": "4.1.0", "default_locale": "en-US", "applications": { "gecko": { diff --git a/modules/AiClassifier.js b/modules/AiClassifier.js index 51951e6..603816f 100644 --- a/modules/AiClassifier.js +++ b/modules/AiClassifier.js @@ -97,14 +97,7 @@ export function getCachedDetails(key, threshold = config.decisionThreshold) { } export function getCachedResult(key, threshold) { return getCachedDetails(key, threshold)?.matched ?? null; } export function getReason(key) { return cache.get(key)?.reason ?? null; } -export async function classify(text, criterion, { cacheKey = null, snapshot = getConfig(), threshold = snapshot.decisionThreshold } = {}) { - await init(); - if (snapshot.apiType === 'systemone') validateThreshold(threshold); - const cached = getCachedDetails(cacheKey, threshold); - if (cached) return cached; - const epoch = generation; - const adapter = adapters[snapshot.apiType]; - const payload = adapter.buildRequest(text, criterion, snapshot); +async function sendRequest(payload, snapshot) { try { await storage.local.set({ lastPayload: payload }); } catch (error) { aiLog('Failed to save last payload', { level: 'warn' }, error); } const headers = { 'Content-Type': 'application/json' }; @@ -126,7 +119,18 @@ export async function classify(text, criterion, { cacheKey = null, snapshot = ge ? ' Verify that the running server build supports /v1/systemone and the selected decision model.' : ''; throw new Error(`Classification request failed: HTTP ${response.status} at ${endpoint}.${detail}${hint}`); } - const raw = await response.json(); + return response.json(); +} + +export async function classify(text, criterion, { cacheKey = null, snapshot = getConfig(), threshold = snapshot.decisionThreshold } = {}) { + await init(); + if (snapshot.apiType === 'systemone') validateThreshold(threshold); + const cached = getCachedDetails(cacheKey, threshold); + if (cached) return cached; + const epoch = generation; + const adapter = adapters[snapshot.apiType]; + const payload = adapter.buildRequest(text, criterion, snapshot); + const raw = await sendRequest(payload, snapshot); const entry = { ...adapter.parseResponse(raw), backend: snapshot.apiType, model: typeof raw.model === 'string' ? raw.model : snapshot.model, version: 2, messageHash: cacheKey?.split(':')[1] }; @@ -136,6 +140,60 @@ export async function classify(text, criterion, { cacheKey = null, snapshot = ge } return withPolicy(entry, threshold); } +// Yield lazily so the action runner can stop without starting the next request. +// Candidates have precomputed keys, allowing actions to move/delete the message. +export async function* classifyRules(text, candidates, snapshot = getConfig()) { + await init(); + validateSettings(snapshot); + if (snapshot.apiType !== 'systemone') { + for (const candidate of candidates) { + const result = await classify(text, candidate.rule.criterion, { + cacheKey: candidate.cacheKey, snapshot, + threshold: candidate.rule.decisionThreshold ?? snapshot.decisionThreshold + }); + yield { rule: candidate.rule, result }; + } + return; + } + const limit = snapshot.systemOneProcessAll ? Infinity : snapshot.systemOneBatchSize; + // Keep received evidence locally even if persistence is cleared during this run. + const evidence = new Map(); + for (const candidate of candidates) { + validateThreshold(candidate.rule.decisionThreshold ?? snapshot.decisionThreshold); + const entry = cache.get(candidate.cacheKey); + if (entry) evidence.set(candidate.cacheKey, entry); + } + for (let index = 0; index < candidates.length; index++) { + const candidate = candidates[index]; + if (!evidence.has(candidate.cacheKey)) { + const pending = new Map(); + for (let next = index; next < candidates.length && pending.size < limit; next++) { + const item = candidates[next]; + if (!evidence.has(item.cacheKey) && !pending.has(item.cacheKey)) { + pending.set(item.cacheKey, { ...item, criterion: item.rule.criterion }); + } + } + const questions = [...pending.values()]; + const epoch = generation; + const raw = await sendRequest(systemone.buildBatchRequest(text, questions, snapshot), snapshot); + // Parse the complete batch before accepting any of its evidence. + const parsed = systemone.parseBatchResponse(raw, questions); + const entries = questions.map((question, i) => [question.cacheKey, { + ...parsed[i], backend: snapshot.apiType, version: 2, + model: typeof raw.model === 'string' ? raw.model : snapshot.model, + messageHash: question.cacheKey.split(':')[1] + }]); + for (const [key, entry] of entries) evidence.set(key, entry); + if (epoch === generation) { + for (const [key, entry] of entries) cache.set(key, entry); + await saveCache(); + } + } + yield { rule: candidate.rule, result: withPolicy(evidence.get(candidate.cacheKey), + candidate.rule.decisionThreshold ?? snapshot.decisionThreshold) }; + } +} + export async function classifyText(text, criterion, cacheKey = null) { return (await classify(text, criterion, { cacheKey })).matched; } export async function removeCacheEntries(keys = []) { await init(); generation++; diff --git a/modules/backends/systemOne.js b/modules/backends/systemOne.js index 0de2472..70627f1 100644 --- a/modules/backends/systemOne.js +++ b/modules/backends/systemOne.js @@ -1,17 +1,22 @@ export const path = '/v1/systemone'; export function buildRequest(text, criterion, config) { - const payload = { - state: text, - questions: { match: { type: 'noul', instructions: `Does this email satisfy the following criterion: ${criterion}` } } - }; + return buildBatchRequest(text, [{ id: 'match', criterion }], config); +} +export function buildBatchRequest(text, questions, config) { + const payload = { state: text, questions: Object.fromEntries(questions.map(({ id, criterion }) => + [id, { type: 'noul', instructions: `Does this email satisfy the following criterion: ${criterion}` }])) }; if (config.model) payload.model = config.model; return payload; } -export function parseResponse(result) { - const answer = result?.answers?.match; +export function parseResponse(result) { return parseAnswer(result, 'match'); } +export function parseBatchResponse(result, questions) { + return questions.map(({ id }) => parseAnswer(result, id)); +} +function parseAnswer(result, id) { + const answer = result?.answers?.[id]; if (answer?.type !== 'noul' || typeof answer.noul !== 'number' || !Number.isFinite(answer.noul) || answer.noul < 0 || answer.noul > 1) { - throw new Error('SystemOne response must contain answers.match with type noul and a probability from 0 to 1.'); + throw new Error(`SystemOne response must contain answers.${id} with type noul and a probability from 0 to 1.`); } return { matched: null, reason: null, probability: answer.noul }; } diff --git a/modules/classificationConfig.js b/modules/classificationConfig.js index 3abe7cc..d6dbb5a 100644 --- a/modules/classificationConfig.js +++ b/modules/classificationConfig.js @@ -2,6 +2,7 @@ import { DEFAULT_AI_PARAMS } from './defaultParams.js'; export const CONFIG_DEFAULTS = { apiType: 'chat-completions', endpoint: 'http://127.0.0.1:5000', model: '', apiKey: '', openaiOrganization: '', openaiProject: '', customSystemPrompt: '', + systemOneProcessAll: true, systemOneBatchSize: 8, decisionThreshold: 0.5, inputCharacterLimit: 3686, aiParams: DEFAULT_AI_PARAMS, htmlToMarkdown: false, stripUrlParams: false, altTextImages: false, collapseWhitespace: false, tokenReduction: false, debugLogging: false @@ -13,6 +14,12 @@ export function validateThreshold(value) { return value; } export function validateSettings(data) { + if (data.systemOneProcessAll !== undefined && typeof data.systemOneProcessAll !== 'boolean') { + throw new Error('Process all must be a boolean.'); + } + if (data.systemOneBatchSize !== undefined && (!Number.isSafeInteger(data.systemOneBatchSize) || data.systemOneBatchSize < 1)) { + throw new Error('Rules per request must be a positive whole number.'); + } if (data.apiType !== undefined && !['chat-completions', 'systemone'].includes(data.apiType)) throw new Error('Unknown API type.'); if (data.decisionThreshold !== undefined) validateThreshold(data.decisionThreshold); if (data.inputCharacterLimit !== undefined && (!Number.isSafeInteger(data.inputCharacterLimit) || data.inputCharacterLimit < 1)) { diff --git a/options/dataTransfer.js b/options/dataTransfer.js index ef2b237..298974c 100644 --- a/options/dataTransfer.js +++ b/options/dataTransfer.js @@ -4,6 +4,7 @@ import { validCacheEntry } from '../modules/AiClassifier.js'; const storage = (globalThis.messenger ?? browser).storage; const KEY_GROUPS = { settings: [ + 'systemOneProcessAll', 'systemOneBatchSize', 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'tokenReduction', 'endpoint', 'model', diff --git a/options/options.html b/options/options.html index 8091df5..2ab0c8a 100644 --- a/options/options.html +++ b/options/options.html @@ -78,6 +78,15 @@
    +
    + +

    Group all eligible, uncached rules for each email into one request. Actions still follow rule order and stop-after-match.

    +
    +
    + + +

    Turn off Process all to limit each request. Use 1 for one rule at a time.

    +
    diff --git a/options/options.js b/options/options.js index d1f71a2..124d163 100644 --- a/options/options.js +++ b/options/options.js @@ -7,6 +7,7 @@ document.addEventListener('DOMContentLoaded', async () => { const { DEFAULT_AI_PARAMS } = await import(browser.runtime.getURL('modules/defaultParams.js')); const { validateSettings, migratedInputLimit } = await import(browser.runtime.getURL('modules/classificationConfig.js')); const defaults = await storage.local.get([ + 'systemOneProcessAll', 'systemOneBatchSize', 'apiType', 'decisionThreshold', 'inputCharacterLimit', 'endpoint', 'customSystemPrompt', @@ -90,6 +91,15 @@ document.addEventListener('DOMContentLoaded', async () => { }); const apiTypeInput = document.getElementById('api-type'); apiTypeInput.value = defaults.apiType || 'chat-completions'; + const processAllInput = document.getElementById('systemone-process-all'); + const batchSizeInput = document.getElementById('systemone-batch-size'); + processAllInput.checked = defaults.systemOneProcessAll ?? true; + let savedBatchSize = defaults.systemOneBatchSize ?? 8; + batchSizeInput.value = savedBatchSize; + function updateBatchControls() { + batchSizeInput.disabled = processAllInput.checked || apiTypeInput.value !== 'systemone'; + } + processAllInput.addEventListener('change', updateBatchControls); const thresholdInput = document.getElementById('decision-threshold'); thresholdInput.value = (defaults.decisionThreshold ?? 0.5) * 100; const inputLimit = document.getElementById('input-character-limit'); @@ -97,6 +107,7 @@ document.addEventListener('DOMContentLoaded', async () => { const settingsError = document.getElementById('settings-error'); function updateApiControls() { const decision = apiTypeInput.value === 'systemone'; + updateBatchControls(); document.querySelectorAll('.systemone-only').forEach(el => el.classList.toggle('is-hidden', !decision)); for (const id of ['system-instructions', 'openai-organization', 'openai-project', ...Object.keys(DEFAULT_AI_PARAMS)]) { document.getElementById(id)?.closest('.field')?.classList.toggle('is-hidden', decision); @@ -1097,6 +1108,14 @@ document.addEventListener('DOMContentLoaded', async () => { document.getElementById('save').addEventListener('click', async () => { settingsError.textContent = ''; const apiType = apiTypeInput.value; + const systemOneProcessAll = processAllInput.checked; + const enteredBatchSize = Number(batchSizeInput.value); + const validBatchSize = Number.isSafeInteger(enteredBatchSize) && enteredBatchSize > 0; + if (!batchSizeInput.disabled && !validBatchSize) { + settingsError.textContent = 'Rules per request must be a positive whole number.'; + return; + } + const systemOneBatchSize = validBatchSize ? enteredBatchSize : savedBatchSize; const decisionThreshold = thresholdInput.value.trim() === '' ? NaN : Number(thresholdInput.value) / 100; const inputCharacterLimit = Number(inputLimit.value); const endpoint = endpointInput.value.trim(); @@ -1127,13 +1146,15 @@ document.addEventListener('DOMContentLoaded', async () => { const tokenReduction = tokenReductionToggle.checked; const showDebugTab = debugTabToggle.checked; const theme = themeSelect.value; - try { validateSettings({ apiType, decisionThreshold, inputCharacterLimit, aiRules: rules }); } + try { validateSettings({ systemOneProcessAll, systemOneBatchSize, apiType, decisionThreshold, inputCharacterLimit, aiRules: rules }); } catch (error) { settingsError.textContent = error.message; return; } - await storage.local.set({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + await storage.local.set({ systemOneProcessAll, systemOneBatchSize, apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging, htmlToMarkdown, stripUrlParams, altTextImages, collapseWhitespace, tokenReduction, aiRules: rules, theme, showDebugTab }); + savedBatchSize = systemOneBatchSize; + batchSizeInput.value = savedBatchSize; await storage.local.remove(['templateName', 'customTemplate']); await applyTheme(theme); try { - await AiClassifier.setConfig({ apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); + await AiClassifier.setConfig({ systemOneProcessAll, systemOneBatchSize, apiType, decisionThreshold, inputCharacterLimit, endpoint, model, apiKey, openaiOrganization, openaiProject, customSystemPrompt, aiParams: aiParamsSave, debugLogging }); logger.setDebug(debugLogging); } catch (e) { logger.aiLog('[options] failed to apply config', {level: 'error'}, e); diff --git a/tests/batching.test.mjs b/tests/batching.test.mjs new file mode 100644 index 0000000..000ebe3 --- /dev/null +++ b/tests/batching.test.mjs @@ -0,0 +1,219 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import { readFile } from 'node:fs/promises'; +import { CONFIG_DEFAULTS, validateSettings } from '../modules/classificationConfig.js'; +import * as systemone from '../modules/backends/systemOne.js'; + +const data = {}; +let cacheWrites = 0; +globalThis.messenger = { storage: { local: { + async get(keys) { return Object.fromEntries((Array.isArray(keys) ? keys : [keys]).map(key => [key, data[key]])); }, + async set(values) { Object.assign(data, structuredClone(values)); if ('aiCache' in values) cacheWrites++; }, + async remove() {} +} }, messages: { async get(id) { return { headerMessageId: `mail-${id}` }; } } }; +const classifier = await import('../modules/AiClassifier.js'); +const transfer = await import('../options/dataTransfer.js'); +let requests = []; +async function setup(update = {}) { + await classifier.clearCache(); + classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', ...update }); + requests = []; cacheWrites = 0; + globalThis.fetch = async (url, request) => { + const body = JSON.parse(request.body); + requests.push({ url, body, headers: request.headers }); + // Intentionally reverse response order and include an unsolicited answer. + return { ok: true, json: async () => ({ model: 'served', answers: { + unsolicited: { type: 'noul', noul: 1 }, + ...Object.fromEntries(Object.keys(body.questions).reverse().map(id => [id, { type: 'noul', noul: 0.75 }])) + } }) }; + }; + return classifier.getConfig(); +} +async function candidates(count, snapshot, criteria = []) { + return Promise.all(Array.from({ length: count }, async (_, i) => { + const criterion = criteria[i] ?? `criterion ${i}`; + return { id: `rule_${i}`, rule: { criterion, actions: [] }, cacheKey: await classifier.buildCacheKey(1, criterion, snapshot) }; + })); +} +async function collect(items, snapshot) { + const results = []; + for await (const value of classifier.classifyRules('email state', items, snapshot)) results.push(value); + return results; +} + +test('Process all, explicit batch sizes, partial batches, empty input and cache reuse', async () => { + for (const [settings, expected] of [ + [{}, [10]], [{ systemOneProcessAll: false, systemOneBatchSize: 8 }, [8, 2]], + [{ systemOneProcessAll: false, systemOneBatchSize: 1 }, Array(10).fill(1)] + ]) { + const snapshot = await setup(settings); + const items = await candidates(10, snapshot); + const results = await collect(items, snapshot); + assert.deepEqual(requests.map(r => Object.keys(r.body.questions).length), expected); + assert.equal(cacheWrites, expected.length); + assert.deepEqual(results.map(r => r.rule.criterion), items.map(r => r.rule.criterion)); + assert.ok(results.every(r => r.result.probability === 0.75)); + assert.ok(requests.every(r => r.body.state === 'email state' && !('messages' in r.body))); + await collect(items, snapshot); + assert.equal(requests.length, expected.length); + await collect([], snapshot); + assert.equal(requests.length, expected.length); + } +}); + +test('cached rules consume no capacity and duplicate criteria use independent thresholds', async () => { + const snapshot = await setup({ systemOneProcessAll: false, systemOneBatchSize: 2 }); + const items = await candidates(5, snapshot, ['cached', 'same', 'cached', 'other', 'same']); + await collect([items[0]], snapshot); + requests = []; cacheWrites = 0; + items[1].rule.decisionThreshold = 0.8; + items[4].rule.decisionThreshold = 0.7; + const results = await collect(items, snapshot); + assert.equal(requests.length, 1); + assert.deepEqual(Object.keys(requests[0].body.questions), ['rule_1', 'rule_3']); + assert.equal(results[1].result.matched, false); + assert.equal(results[4].result.matched, true); + assert.equal(cacheWrites, 1); + assert.equal(await classifier.buildCacheKey(1, 'same', { ...snapshot, systemOneProcessAll: true, systemOneBatchSize: 100 }), items[1].cacheKey); +}); + +test('answer identity determines probability, not answer order', () => { + const result = { answers: { rule_2: { type: 'noul', noul: 0 }, rule_1: { type: 'noul', noul: 1 } } }; + assert.deepEqual(systemone.parseBatchResponse(result, [{ id: 'rule_1' }, { id: 'rule_2' }]).map(r => r.probability), [1, 0]); +}); + +test('failed batches neither yield nor cache partial results and do not retry', async () => { + for (const failure of ['missing', 'invalid', 'http']) { + const snapshot = await setup(); + const items = await candidates(2, snapshot); + let calls = 0, yielded = 0; + globalThis.fetch = async () => { + calls++; + return { ok: failure !== 'http', status: 413, json: async () => ({ answers: { + rule_0: { type: 'noul', noul: 0.9 }, + ...(failure === 'invalid' ? { rule_1: { type: 'noul', noul: '0.9' } } : {}) + } }) }; + }; + await assert.rejects(async () => { + for await (const value of classifier.classifyRules('state', items, snapshot)) yielded++; + }); + assert.equal(yielded, 0); + assert.equal(calls, 1); + assert.equal(cacheWrites, 0); + assert.equal(await classifier.getCacheSize(), 0); + } +}); + +test('clearing or importing during a batch prevents stale cache writes', async () => { + for (const replace of [() => classifier.clearCache(), () => classifier.replaceCache({})]) { + const snapshot = await setup(); + const items = await candidates(2, snapshot); + const fetch = globalThis.fetch; + globalThis.fetch = async (...args) => { await replace(); return fetch(...args); }; + const results = await collect(items, snapshot); + assert.equal(results.length, 2); + assert.equal(requests.length, 1); + assert.equal(await classifier.getCacheSize(), 0); + } +}); + +test('Chat Completions remains lazy and sequential', async () => { + const snapshot = await setup({ apiType: 'chat-completions' }); + const items = await candidates(3, snapshot); + globalThis.fetch = async (url, request) => { + requests.push(JSON.parse(request.body)); + return { ok: true, json: async () => ({ choices: [{ message: { content: '{"match":true}' } }] }) }; + }; + for await (const value of classifier.classifyRules('state', items, snapshot)) break; + assert.equal(requests.length, 1); + assert.ok(Array.isArray(requests[0].messages)); +}); + +test('batch settings validate and survive import', async () => { + for (const value of [0, -1, 1.2, Infinity, '8', null]) assert.throws(() => validateSettings({ systemOneBatchSize: value })); + assert.throws(() => validateSettings({ systemOneProcessAll: 'true' })); + await transfer.importData({ text: async () => JSON.stringify({ systemOneProcessAll: false, systemOneBatchSize: 3 }) }, ['settings']); + assert.equal(data.systemOneProcessAll, false); + assert.equal(data.systemOneBatchSize, 3); + await assert.rejects(transfer.importData({ text: async () => '{"systemOneBatchSize":0}' }, ['settings'])); +}); + +// Exercise the action runner with the real lazy classifier and mocked mail APIs. +const source = await readFile(new URL('../background.js', import.meta.url), 'utf8'); +const functions = source.slice(source.indexOf('async function processMessage('), source.indexOf('async function clearCacheForMessages(')); +test('stop after match skips later actions and batches; all-mode retains later evidence', async () => { + for (const processAll of [true, false]) { + const snapshot = await setup({ systemOneProcessAll: processAll, systemOneBatchSize: 2 }); + const actions = [], errors = []; + const rules = Array.from({ length: 5 }, (_, i) => ({ criterion: `rule ${i}`, actions: [{ type: 'flag', flagged: true }], stopProcessing: i === 1 })); + const context = vm.createContext({ + structuredClone, Date, Error, aiRules: rules, AiClassifier: classifier, + queue: Promise.resolve(), queuedCount: 0, processing: false, currentStart: 0, + tokenReduction: false, showDebugTab: false, timingStats: {}, + updateActionIcon() {}, updateTimingStats() {}, showTransientIcon() {}, + buildEmailText() { return 'state'; }, ICONS: { circle: 'circle' }, + logger: { aiLog() {} }, recordError(...args) { errors.push(args); }, ERROR_NOTIFICATION_ID: 'error', + storage: messenger.storage, + messenger: { messages: { + async getFull() { return {}; }, async get() { return { tags: [], folder: { accountId: 'account' } }; }, + async update(id, value) { actions.push({ id, value }); } + }, identities: { async list() { return []; } } }, + browser: { runtime: { getURL: x => x }, notifications: { async create() {} } } + }); + vm.runInContext(functions, context); + await vm.runInContext('applyAiRules([1])', context); + assert.equal(errors.length, 0); + assert.equal(actions.length, 2); + assert.equal(requests.length, 1); + assert.equal(Object.keys(requests[0].body.questions).length, processAll ? 5 : 2); + assert.equal(await classifier.getCacheSize(), processAll ? 5 : 2); + } +}); + +test('batch controls default to all and retain numeric input across toggles and API switches', async () => { + const optionsSource = await readFile(new URL('../options/options.js', import.meta.url), 'utf8'); + const section = optionsSource.slice(optionsSource.indexOf(' const apiTypeInput ='), optionsSource.indexOf(' const endpointInput =')); + const fields = new Map(); + const field = id => { + if (!fields.has(id)) fields.set(id, { + value: '', checked: false, disabled: false, listeners: {}, + addEventListener(event, fn) { this.listeners[event] = fn; }, + classList: { toggle() {} }, closest() { return this; } + }); + return fields.get(id); + }; + const context = vm.createContext({ + defaults: { apiType: 'systemone' }, DEFAULT_AI_PARAMS: {}, migratedInputLimit() { return 3686; }, + updateEndpointPreview() {}, document: { getElementById: field, querySelectorAll() { return []; } } + }); + vm.runInContext(`${section}\nupdateApiControls();`, context); + const all = field('systemone-process-all'), size = field('systemone-batch-size'), api = field('api-type'); + assert.equal(all.checked, true); + assert.equal(size.value, 8); + assert.equal(size.disabled, true); + all.checked = false; all.listeners.change(); + assert.equal(size.disabled, false); + size.value = '3'; + all.checked = true; all.listeners.change(); + assert.equal(size.disabled, true); + all.checked = false; all.listeners.change(); + assert.equal(size.value, '3'); + api.value = 'chat-completions'; api.listeners.change(); + assert.equal(size.disabled, true); + api.value = 'systemone'; api.listeners.change(); + assert.equal(size.disabled, false); + assert.equal(size.value, '3'); +}); + +test('batch settings remain fixed while a per-email snapshot is running', async () => { + const snapshot = await setup({ systemOneProcessAll: false, systemOneBatchSize: 2 }); + const items = await candidates(5, snapshot); + const fetch = globalThis.fetch; + globalThis.fetch = async (...args) => { + classifier.setConfig({ systemOneProcessAll: true, systemOneBatchSize: 1 }); + return fetch(...args); + }; + await collect(items, snapshot); + assert.deepEqual(requests.map(r => Object.keys(r.body.questions).length), [2, 2, 1]); +}); diff --git a/tests/cacheRestore.test.mjs b/tests/cacheRestore.test.mjs index f5ca74d..91a8515 100644 --- a/tests/cacheRestore.test.mjs +++ b/tests/cacheRestore.test.mjs @@ -51,7 +51,7 @@ const messageStart = source.indexOf('browser.runtime.onMessage.addListener('); vm.runInContext(source.slice(messageStart, source.indexOf('\n\n // Automatically classify', messageStart)), context); test('export/reinstall/import restores Details immediately with changed Thunderbird IDs', async () => { - const settings = { ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 }; + const settings = { ...CONFIG_DEFAULTS, systemOneProcessAll: false, systemOneBatchSize: 3, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 }; const rules = [{ criterion: 'invoice', decisionThreshold: 0.8, actions: [] }]; await storage.local.set({ ...settings, aiRules: rules }); globalThis.fetch = async () => ({ ok: true, json: async () => ({ answers: { match: { type: 'noul', noul: 0.9 } } }) }); @@ -73,6 +73,8 @@ test('export/reinstall/import restores Details immediately with changed Thunderb assert.equal((await onMessage({ type: 'sortana:getDetails', id: 99 })).results.length, 0); await transfer.importData(exported, ['settings', 'rules', 'cache']); assert.equal((await onMessage({ type: 'sortana:getCacheSize' })).count, 1); + assert.equal(classifier.getConfig().systemOneProcessAll, false); + assert.equal(classifier.getConfig().systemOneBatchSize, 3); const details = await onMessage({ type: 'sortana:getDetails', id: 99 }); assert.equal(details.results.length, 1); assert.equal(details.results[0].criterion, 'invoice'); diff --git a/tests/ruleExecution.test.mjs b/tests/ruleExecution.test.mjs index 153a050..327a14d 100644 --- a/tests/ruleExecution.test.mjs +++ b/tests/ruleExecution.test.mjs @@ -39,10 +39,13 @@ function runner({ fail = false, changeSettings = false } = {}) { AiClassifier: { getConfig() { return structuredClone(currentConfig); }, async buildCacheKey(id, criterion) { return `${id}:${criterion}`; }, - async classify(text, criterion, options) { - evaluated.push({ text, criterion, ...options }); - if (fail && options.cacheKey.startsWith('1:')) throw new Error('inference failed'); - return { matched: true }; + async *classifyRules(text, candidates, snapshot) { + for (const { rule, cacheKey } of candidates) { + evaluated.push({ text, criterion: rule.criterion, cacheKey, snapshot, + threshold: rule.decisionThreshold ?? snapshot.decisionThreshold }); + if (fail && cacheKey.startsWith('1:')) throw new Error('inference failed'); + yield { rule, result: { matched: true } }; + } } } });