Add SystemOne classification support and release version 4.0.0
This commit is contained in:
parent
76195f9a92
commit
9a5404cb62
16 changed files with 923 additions and 579 deletions
84
tests/cacheRestore.test.mjs
Normal file
84
tests/cacheRestore.test.mjs
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import vm from 'node:vm';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { CONFIG_DEFAULTS } from '../modules/classificationConfig.js';
|
||||
|
||||
const persisted = {};
|
||||
const listeners = [];
|
||||
const storage = {
|
||||
local: {
|
||||
async get(keys) {
|
||||
return Object.fromEntries((Array.isArray(keys) ? keys : [keys])
|
||||
.filter(key => key in persisted).map(key => [key, structuredClone(persisted[key])]));
|
||||
},
|
||||
async set(values) {
|
||||
const changes = {};
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
changes[key] = { oldValue: persisted[key], newValue: structuredClone(value) };
|
||||
persisted[key] = structuredClone(value);
|
||||
}
|
||||
for (const listener of listeners) await listener(changes, 'local');
|
||||
},
|
||||
async remove(keys) {
|
||||
for (const key of Array.isArray(keys) ? keys : [keys]) delete persisted[key];
|
||||
}
|
||||
},
|
||||
onChanged: { addListener(listener) { listeners.push(listener); } }
|
||||
};
|
||||
const messenger = { storage, messages: { async get(id) {
|
||||
// Numeric Thunderbird IDs may change after reinstall; RFC Message-ID does not.
|
||||
return { subject: 'Invoice', headerMessageId: 'persistent-message@example.com' };
|
||||
} } };
|
||||
globalThis.messenger = messenger;
|
||||
const classifier = await import('../modules/AiClassifier.js');
|
||||
const transfer = await import('../options/dataTransfer.js');
|
||||
const source = await readFile(new URL('../background.js', import.meta.url), 'utf8');
|
||||
let onMessage;
|
||||
const context = vm.createContext({
|
||||
storage, messenger, structuredClone, AiClassifier: classifier, aiRules: [],
|
||||
logger: { aiLog() {} }, logGetTiming: false,
|
||||
htmlToMarkdown: false, stripUrlParams: false, altTextImages: false,
|
||||
collapseWhitespace: false, tokenReduction: false, showDebugTab: false,
|
||||
browser: { runtime: { onMessage: { addListener(listener) { onMessage = listener; } } } }
|
||||
});
|
||||
// Exercise the actual background storage and Details handlers, not a manual cache reload.
|
||||
const normalizeStart = source.indexOf('function normalizeRules(');
|
||||
vm.runInContext(source.slice(normalizeStart, source.indexOf('function iconPaths(', normalizeStart)), context);
|
||||
const changesStart = source.indexOf('storage.onChanged.addListener(');
|
||||
vm.runInContext(source.slice(changesStart, source.indexOf('\n\n if (browser.theme', changesStart)), context);
|
||||
const messageStart = source.indexOf('browser.runtime.onMessage.addListener(');
|
||||
vm.runInContext(source.slice(messageStart, source.indexOf('\n\n // Automatically classify', messageStart)), context);
|
||||
|
||||
test('export/reinstall/import restores Details immediately with changed Thunderbird IDs', async () => {
|
||||
const settings = { ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://decision-server:8080', tokenReduction: true, inputCharacterLimit: 5000 };
|
||||
const rules = [{ criterion: 'invoice', decisionThreshold: 0.8, actions: [] }];
|
||||
await storage.local.set({ ...settings, aiRules: rules });
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ answers: { match: { type: 'noul', noul: 0.9 } } }) });
|
||||
const key = await classifier.buildCacheKey(1, 'invoice');
|
||||
await classifier.classify('Invoice attached', 'invoice', { cacheKey: key, threshold: 0.8 });
|
||||
let exported;
|
||||
const originalDocument = globalThis.document;
|
||||
const createURL = URL.createObjectURL;
|
||||
const revokeURL = URL.revokeObjectURL;
|
||||
URL.createObjectURL = blob => { exported = blob; return 'blob:export'; };
|
||||
URL.revokeObjectURL = () => {};
|
||||
globalThis.document = { body: { appendChild() {} }, createElement() { return { click() {}, remove() {} }; } };
|
||||
try { await transfer.exportData(['settings', 'rules', 'cache']); }
|
||||
finally { globalThis.document = originalDocument; URL.createObjectURL = createURL; URL.revokeObjectURL = revokeURL; }
|
||||
await classifier.clearCache();
|
||||
for (const key of Object.keys(persisted)) delete persisted[key];
|
||||
classifier.setConfig(CONFIG_DEFAULTS);
|
||||
context.aiRules = [];
|
||||
assert.equal((await onMessage({ type: 'sortana:getDetails', id: 99 })).results.length, 0);
|
||||
await transfer.importData(exported, ['settings', 'rules', 'cache']);
|
||||
assert.equal((await onMessage({ type: 'sortana:getCacheSize' })).count, 1);
|
||||
const details = await onMessage({ type: 'sortana:getDetails', id: 99 });
|
||||
assert.equal(details.results.length, 1);
|
||||
assert.equal(details.results[0].criterion, 'invoice');
|
||||
assert.equal(details.results[0].probability, 0.9);
|
||||
assert.equal(details.results[0].threshold, 0.8);
|
||||
assert.equal(details.results[0].matched, true);
|
||||
globalThis.fetch = () => { throw new Error('Restored cache must avoid inference'); };
|
||||
assert.equal((await classifier.classify('Invoice attached', 'invoice', { cacheKey: await classifier.buildCacheKey(99, 'invoice') })).matched, true);
|
||||
});
|
||||
167
tests/classification.test.mjs
Normal file
167
tests/classification.test.mjs
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import * as chat from '../modules/backends/chatCompletions.js';
|
||||
import * as decision from '../modules/backends/systemOne.js';
|
||||
import { CONFIG_DEFAULTS, migratedInputLimit, validateSettings } from '../modules/classificationConfig.js';
|
||||
|
||||
const data = { aiCache: { legacy: { matched: true, reason: 'old' } }, aiReasonCache: { legacy: 'old' } };
|
||||
globalThis.messenger = { storage: { local: {
|
||||
async get(keys) { return Object.fromEntries((Array.isArray(keys) ? keys : [keys]).map(key => [key, data[key]])); },
|
||||
async set(values) { Object.assign(data, structuredClone(values)); },
|
||||
async remove(keys) { for (const key of Array.isArray(keys) ? keys : [keys]) delete data[key]; }
|
||||
} }, messages: { async get(id) { return { headerMessageId: `message-${id}` }; } } };
|
||||
const classifier = await import('../modules/AiClassifier.js');
|
||||
const transfer = await import('../options/dataTransfer.js');
|
||||
const answer = probability => ({ model: 'served-model', answers: { match: { type: 'noul', noul: probability } } });
|
||||
let calls = [];
|
||||
function serve(body, status = 200) {
|
||||
globalThis.fetch = async (url, request) => { calls.push({ url, ...request }); return { ok: status === 200, status, json: async () => body }; };
|
||||
}
|
||||
|
||||
test('adapters preserve chat prompts and isolate SystemOne payloads', () => {
|
||||
const config = { ...CONFIG_DEFAULTS, model: 'chosen', customSystemPrompt: 'Custom guidance' };
|
||||
const payload = chat.buildRequest('email', 'invoice', config);
|
||||
assert.match(payload.messages[0].content, /Custom guidance/);
|
||||
assert.match(payload.messages[1].content, /Classification Criterion: invoice/);
|
||||
assert.equal(payload.max_tokens, 4096);
|
||||
assert.deepEqual(decision.buildRequest('email', 'invoice', config), {
|
||||
state: 'email', model: 'chosen', questions: { match: { type: 'noul', instructions: 'Does this email satisfy the following criterion: invoice' } }
|
||||
});
|
||||
assert.equal('model' in decision.buildRequest('email', 'invoice', CONFIG_DEFAULTS), false);
|
||||
assert.equal('model' in chat.buildRequest('email', 'invoice', CONFIG_DEFAULTS), false);
|
||||
});
|
||||
|
||||
test('parsers validate evidence including zero, one, and malformed answers', () => {
|
||||
for (const probability of [0, 0.5, 1]) assert.equal(decision.parseResponse(answer(probability)).probability, probability);
|
||||
for (const probability of [null, '0.5', -0.1, 1.1, Infinity, NaN]) assert.throws(() => decision.parseResponse(answer(probability)));
|
||||
for (const raw of [null, {}, { answers: { other: { type: 'noul', noul: 0.5 } } }, { answers: { match: { type: 'choice', noul: 0.5 } } }]) assert.throws(() => decision.parseResponse(raw));
|
||||
assert.deepEqual(chat.parseResponse({ choices: [{ message: { content: 'text {"match":false} then {"matched":true,"reason":"brace } inside"}' } }] }), { matched: true, reason: 'brace } inside', probability: null });
|
||||
for (const content of ['', '{broken}', '{"match":"true"}']) assert.throws(() => chat.parseResponse({ choices: [{ message: { content } }] }));
|
||||
assert.throws(() => chat.parseResponse(null));
|
||||
});
|
||||
|
||||
test('URLs preserve proxy prefixes and normalize API suffixes', () => {
|
||||
for (const suffix of ['', '/', '/v1', '/v1/', '/v1/models', '/v1/chat/completions/', '/v1/systemone']) {
|
||||
assert.equal(classifier.buildEndpointUrl(`http://localhost:8080/proxy${suffix}`, 'systemone'), 'http://localhost:8080/proxy/v1/systemone');
|
||||
assert.equal(classifier.buildModelsUrl(`http://localhost:8080/proxy${suffix}`), 'http://localhost:8080/proxy/v1/models');
|
||||
}
|
||||
assert.equal(classifier.buildEndpointUrl('example.com'), 'https://example.com/v1/chat/completions');
|
||||
assert.equal(classifier.buildEndpointUrl(''), '');
|
||||
});
|
||||
|
||||
test('settings validation and independent character-limit migration', () => {
|
||||
assert.equal(migratedInputLimit({}), 3686);
|
||||
assert.equal(migratedInputLimit({ aiParams: { max_tokens: 1000 } }), 900);
|
||||
assert.equal(migratedInputLimit({ inputCharacterLimit: 500, aiParams: { max_tokens: 1000 } }), 500);
|
||||
validateSettings({ decisionThreshold: 0, aiRules: [{ decisionThreshold: 1 }] });
|
||||
for (const value of [-1, 2, null, '0.5']) assert.throws(() => validateSettings({ aiRules: [{ decisionThreshold: value }] }));
|
||||
for (const value of [0, 1.2, NaN]) assert.throws(() => validateSettings({ inputCharacterLimit: value }));
|
||||
});
|
||||
|
||||
test('cache migration, inference reuse, fingerprints, clearing and import', async () => {
|
||||
await classifier.init();
|
||||
assert.equal(await classifier.getCacheSize(), 0);
|
||||
assert.equal(data.aiReasonCache, undefined);
|
||||
assert.deepEqual(data.aiCache, {});
|
||||
classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://localhost:8080', apiKey: 'secret', openaiOrganization: 'org', openaiProject: 'project' });
|
||||
const snapshot = classifier.getConfig();
|
||||
const key = await classifier.buildCacheKey(1, 'invoice', snapshot);
|
||||
serve(answer(0.75)); calls = [];
|
||||
const first = await classifier.classify('email', 'invoice', { snapshot, cacheKey: key, threshold: 0.75 });
|
||||
assert.equal(first.matched, true);
|
||||
assert.equal(first.model, 'served-model');
|
||||
assert.deepEqual(calls[0].headers, { 'Content-Type': 'application/json', Authorization: 'Bearer secret' });
|
||||
const second = await classifier.classify('email', 'invoice', { snapshot, cacheKey: key, threshold: 0.8 });
|
||||
assert.equal(second.matched, false);
|
||||
assert.equal(second.probability, 0.75);
|
||||
assert.equal(calls.length, 1);
|
||||
classifier.replaceCache(data.aiCache);
|
||||
assert.equal(classifier.getCachedDetails(key, 0.75).matched, true);
|
||||
assert.equal(await classifier.buildCacheKey(1, 'invoice', { ...snapshot, decisionThreshold: 0.9, apiKey: 'new secret' }), key);
|
||||
for (const change of [{ apiType: 'chat-completions' }, { endpoint: 'http://other' }, { model: 'other' }, { tokenReduction: true }, { inputCharacterLimit: 500 }]) {
|
||||
assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...snapshot, ...change }), key);
|
||||
}
|
||||
const chatSnapshot = { ...snapshot, apiType: 'chat-completions' };
|
||||
const chatKey = await classifier.buildCacheKey(1, 'invoice', chatSnapshot);
|
||||
assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...chatSnapshot, customSystemPrompt: 'other' }), chatKey);
|
||||
assert.notEqual(await classifier.buildCacheKey(1, 'invoice', { ...chatSnapshot, aiParams: { ...snapshot.aiParams, temperature: 1 } }), chatKey);
|
||||
serve({ choices: [{ message: { content: '{"match":false,"reason":"not invoice"}' } }] });
|
||||
await classifier.classify('email', 'invoice', { cacheKey: chatKey, snapshot: chatSnapshot });
|
||||
assert.equal(calls.at(-1).headers['OpenAI-Organization'], 'org');
|
||||
assert.equal(await classifier.getCacheSize(), 2);
|
||||
const exported = structuredClone(data.aiCache);
|
||||
await classifier.clearCacheForMessage(1);
|
||||
assert.equal(await classifier.getCacheSize(), 0);
|
||||
await transfer.importData({ text: async () => JSON.stringify({ aiCache: { ...exported, legacy: true } }) }, ['cache']);
|
||||
classifier.replaceCache(data.aiCache); // background storage listener consumes imported replacement
|
||||
assert.equal(await classifier.getCacheSize(), 2);
|
||||
await assert.rejects(transfer.importData({ text: async () => JSON.stringify({ decisionThreshold: 2 }) }, ['settings']));
|
||||
await assert.rejects(transfer.importData({ text: async () => JSON.stringify({ aiRules: [{ decisionThreshold: -1 }] }) }, ['rules']));
|
||||
await classifier.clearCache();
|
||||
});
|
||||
|
||||
test('failures remain uncached and snapshots survive configuration changes', async () => {
|
||||
classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone' });
|
||||
const snapshot = classifier.getConfig();
|
||||
const key = await classifier.buildCacheKey(2, 'invoice', snapshot);
|
||||
for (const raw of [null, answer('0.9'), {}]) {
|
||||
serve(raw);
|
||||
await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }));
|
||||
assert.equal(classifier.getCachedDetails(key), null);
|
||||
}
|
||||
serve({}, 503);
|
||||
await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /503/);
|
||||
globalThis.fetch = async () => { throw new Error('network'); };
|
||||
await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /network/);
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => { throw new SyntaxError('bad JSON'); } });
|
||||
await assert.rejects(classifier.classify('email', 'invoice', { cacheKey: key, snapshot }), /bad JSON/);
|
||||
classifier.setConfig({ apiType: 'chat-completions', endpoint: 'http://changed' });
|
||||
serve(answer(1)); calls = [];
|
||||
const result = await classifier.classify('email', 'invoice', { cacheKey: key, snapshot });
|
||||
assert.equal(result.backend, 'systemone');
|
||||
assert.equal(calls[0].url, 'http://127.0.0.1:5000/v1/systemone');
|
||||
assert.equal('Authorization' in calls[0].headers, false);
|
||||
await classifier.clearCache();
|
||||
});
|
||||
|
||||
test('clearing or replacing cache during inference prevents stale repopulation', async () => {
|
||||
classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone' });
|
||||
const snapshot = classifier.getConfig();
|
||||
const key = await classifier.buildCacheKey(3, 'invoice', snapshot);
|
||||
for (const replace of [() => classifier.clearCache(), () => classifier.replaceCache({})]) {
|
||||
let release, started;
|
||||
const ready = new Promise(resolve => { started = resolve; });
|
||||
globalThis.fetch = async () => {
|
||||
started();
|
||||
await new Promise(resolve => { release = resolve; });
|
||||
return { ok: true, json: async () => answer(0.9) };
|
||||
};
|
||||
const pending = classifier.classify('email', 'invoice', { cacheKey: key, snapshot });
|
||||
await ready;
|
||||
await replace();
|
||||
release();
|
||||
assert.equal((await pending).matched, true);
|
||||
assert.equal(classifier.getCachedDetails(key), null);
|
||||
}
|
||||
});
|
||||
|
||||
test('settings and rule imports retain thresholds and reject invalid caches', async () => {
|
||||
const settings = { apiType: 'systemone', decisionThreshold: 0.75, inputCharacterLimit: 5000, tokenReduction: true };
|
||||
const rules = [{ criterion: 'invoice', decisionThreshold: 0, actions: [{ type: 'flag', flagged: true }] }, { criterion: 'urgent', actions: [] }];
|
||||
await transfer.importData({ text: async () => JSON.stringify({ ...settings, aiRules: rules }) }, ['settings', 'rules']);
|
||||
for (const [key, value] of Object.entries(settings)) assert.equal(data[key], value);
|
||||
assert.deepEqual(data.aiRules, rules);
|
||||
const key = await classifier.buildCacheKey(4, 'invoice');
|
||||
assert.equal(classifier.validCacheEntry(key, { version: 2, backend: 'systemone', probability: '0.8', reason: null, messageHash: key.split(':')[1], model: '' }), false);
|
||||
});
|
||||
|
||||
test('SystemOne 404 reports endpoint, server message, and compatibility guidance', async () => {
|
||||
classifier.setConfig({ ...CONFIG_DEFAULTS, apiType: 'systemone', endpoint: 'http://seinei.wageshouse:5003' });
|
||||
serve({ error: { message: 'File Not Found' } }, 404);
|
||||
await assert.rejects(classifier.classify('synthetic email', 'invoice'), error => {
|
||||
assert.match(error.message, /HTTP 404 at http:\/\/seinei\.wageshouse:5003\/v1\/systemone/);
|
||||
assert.match(error.message, /Server response: File Not Found/);
|
||||
assert.match(error.message, /running server build supports/);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
80
tests/ruleExecution.test.mjs
Normal file
80
tests/ruleExecution.test.mjs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import vm from 'node:vm';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
|
||||
// Run the actual background rule/queue functions with Thunderbird services mocked.
|
||||
const source = await readFile(new URL('../background.js', import.meta.url), 'utf8');
|
||||
const functions = source.slice(source.indexOf('async function processMessage('), source.indexOf('async function clearCacheForMessages('));
|
||||
function runner({ fail = false, changeSettings = false } = {}) {
|
||||
const actions = [], evaluated = [], errors = [];
|
||||
const rules = [
|
||||
{ criterion: 'first', decisionThreshold: 0.8, actions: [{ type: 'flag', flagged: true }], stopProcessing: true },
|
||||
{ criterion: 'second', actions: [{ type: 'read', read: true }] }
|
||||
];
|
||||
let currentConfig = { apiType: 'systemone', inputCharacterLimit: 5, decisionThreshold: 0.5 };
|
||||
const context = vm.createContext({
|
||||
structuredClone, Date, Error,
|
||||
aiRules: rules, queue: Promise.resolve(), queuedCount: 0, processing: false,
|
||||
currentStart: 0, tokenReduction: true, showDebugTab: false, timingStats: {},
|
||||
updateActionIcon() {}, updateTimingStats() {}, showTransientIcon() {},
|
||||
buildEmailText() { return 'email contents'; }, ICONS: { circle: 'circle' },
|
||||
logger: { aiLog() {} }, recordError(...args) { errors.push(args); },
|
||||
ERROR_NOTIFICATION_ID: 'error',
|
||||
storage: { local: { async set() {} } },
|
||||
messenger: {
|
||||
messages: {
|
||||
async getFull() { return {}; },
|
||||
async get() {
|
||||
if (changeSettings) {
|
||||
rules[0].decisionThreshold = 0.1;
|
||||
currentConfig = { apiType: 'chat-completions', decisionThreshold: 0.1 };
|
||||
}
|
||||
return { tags: [], folder: { accountId: 'account' } };
|
||||
},
|
||||
async update(id, action) { actions.push({ id, ...action }); }
|
||||
}, identities: { async list() { return []; } }
|
||||
},
|
||||
browser: { runtime: { getURL: x => x }, notifications: { async create() {} } },
|
||||
AiClassifier: {
|
||||
getConfig() { return structuredClone(currentConfig); },
|
||||
async buildCacheKey(id, criterion) { return `${id}:${criterion}`; },
|
||||
async classify(text, criterion, options) {
|
||||
evaluated.push({ text, criterion, ...options });
|
||||
if (fail && options.cacheKey.startsWith('1:')) throw new Error('inference failed');
|
||||
return { matched: true };
|
||||
}
|
||||
}
|
||||
});
|
||||
vm.runInContext(functions, context);
|
||||
return { context, actions, evaluated, errors };
|
||||
}
|
||||
|
||||
test('failure stops remaining rules for one email, next queued email still acts', async () => {
|
||||
const { context, actions, evaluated, errors } = runner({ fail: true });
|
||||
await vm.runInContext('applyAiRules([1, 2])', context);
|
||||
assert.deepEqual(evaluated.map(x => x.cacheKey), ['1:first', '2:first']);
|
||||
assert.deepEqual(actions, [{ id: 2, flagged: true }]);
|
||||
assert.equal(errors.length, 1);
|
||||
assert.equal(context.processing, false);
|
||||
assert.equal(context.queuedCount, 0);
|
||||
});
|
||||
|
||||
test('settings and rules remain fixed during an email; reduction uses characters', async () => {
|
||||
const { context, evaluated, actions } = runner({ changeSettings: true });
|
||||
await vm.runInContext('applyAiRules([1])', context);
|
||||
assert.equal(evaluated[0].text, 'email');
|
||||
assert.equal(evaluated[0].threshold, 0.8);
|
||||
assert.equal(evaluated[0].snapshot.apiType, 'systemone');
|
||||
assert.equal(evaluated.length, 1); // stop after first match
|
||||
assert.equal(actions.length, 1);
|
||||
});
|
||||
|
||||
test('inherited thresholds and eligibility preserve ordered rule processing', async () => {
|
||||
const { context, evaluated, actions } = runner();
|
||||
context.aiRules[0].enabled = false;
|
||||
await vm.runInContext('applyAiRules([1])', context);
|
||||
assert.equal(evaluated[0].criterion, 'second');
|
||||
assert.equal(evaluated[0].threshold, 0.5);
|
||||
assert.deepEqual(actions, [{ id: 1, read: true }]);
|
||||
});
|
||||
Loading…
Reference in a new issue