Split site definitions and generate default collection

README.md

@@ -113,6 +113,13 @@ npm run typecheck
 npm test
 ```
 
+The bundled definition collection is generated from `src/content/definitions/collection.json` and the per-site files in `src/content/definitions/sites/`. Regenerate it after changing a site definition, then commit the resulting `default.json`:
+
+```sh
+npm run definitions:generate
+npm run definitions:check
+```
+
 Build the browser-specific packages with:
 
 ```sh

build.sh

@@ -62,6 +62,10 @@ fi
 
 version="$firefox_version"
 
+echo "Generating site definitions..."
+(cd "$script_dir" && npm run definitions:generate)
+(cd "$script_dir" && npm run definitions:check)
+
 echo "Building Firefox package..."
 (cd "$script_dir" && npm run build:firefox)
 echo "Building Chromium package..."

package.json

@@ -5,6 +5,10 @@
   "description": "A local-first browser extension that hides toxic social-media posts.",
   "type": "module",
   "scripts": {
+    "definitions:generate": "node tools/generate-definitions.mjs",
+    "definitions:check": "node tools/generate-definitions.mjs --check",
+    "prebuild:firefox": "npm run definitions:generate && npm run definitions:check",
+    "prebuild:chromium": "npm run definitions:generate && npm run definitions:check",
     "build": "npm run build:chromium",
     "build:firefox": "vite build --mode firefox && VIBEGUARD_CLASSIC_ENTRY=content vite build --mode firefox && VIBEGUARD_CLASSIC_ENTRY=background vite build --mode firefox",
     "build:chromium": "vite build --mode chromium && VIBEGUARD_CLASSIC_ENTRY=content vite build --mode chromium",

src/content/definitions/collection.json

@@ -0,0 +1,6 @@
+{
+  "schemaVersion": 1,
+  "collectionId": "vibeguard-default",
+  "name": "VibeGuard default definitions",
+  "description": "Definitions maintained by the VibeGuard project."
+}

src/content/definitions/default.json

@@ -7,14 +7,34 @@
     {
       "id": "mastodon",
       "name": "Mastodon (official web interface)",
-      "urlPatterns": ["<all_urls>"],
-      "requiredSelectors": [".status__content", ".status, .detailed-status"],
+      "urlPatterns": [
+        "<all_urls>"
+      ],
+      "requiredSelectors": [
+        ".status__content",
+        ".status, .detailed-status"
+      ],
       "post": {
-        "rootSelectors": [".status", ".detailed-status"],
-        "textSelectors": [".status__content", ".e-content"],
-        "excludedSelectors": [".status__content__spoiler-link", ".status__content__read-more-button"],
-        "idAttributes": ["data-id"],
-        "permalinkSelectors": ["a.status__relative-time[href]", "a.detailed-status__datetime[href]", "a.u-url.u-uid[href]"]
+        "rootSelectors": [
+          ".status",
+          ".detailed-status"
+        ],
+        "textSelectors": [
+          ".status__content",
+          ".e-content"
+        ],
+        "excludedSelectors": [
+          ".status__content__spoiler-link",
+          ".status__content__read-more-button"
+        ],
+        "idAttributes": [
+          "data-id"
+        ],
+        "permalinkSelectors": [
+          "a.status__relative-time[href]",
+          "a.detailed-status__datetime[href]",
+          "a.u-url.u-uid[href]"
+        ]
       }
     },
     {
@@ -26,12 +46,31 @@
         "https://threads.net/*",
         "https://www.threads.net/*"
       ],
-      "requiredSelectors": ["[role=\"article\"], article, div[data-pagelet^=\"threads_post_page_\"]"],
+      "requiredSelectors": [
+        "[role=\"article\"], article, [data-pressable-container=\"true\"], div[data-pagelet^=\"threads_post_page_\"]"
+      ],
       "post": {
-        "rootSelectors": ["[role=\"article\"]", "article", "div[data-pagelet^=\"threads_post_page_\"]"],
-        "textSelectors": ["span[dir=\"auto\"]"],
-        "excludedSelectors": ["button", "[role=\"button\"]", "time", "svg", "a span", "time span", "[role=\"button\"] span"],
-        "permalinkSelectors": ["a[href*=\"/post/\"]"]
+        "rootSelectors": [
+          "[role=\"article\"]",
+          "article",
+          "[data-pressable-container=\"true\"]",
+          "div[data-pagelet^=\"threads_post_page_\"]"
+        ],
+        "textSelectors": [
+          "span[dir=\"auto\"]"
+        ],
+        "excludedSelectors": [
+          "button",
+          "[role=\"button\"]",
+          "time",
+          "svg",
+          "a span",
+          "time span",
+          "[role=\"button\"] span"
+        ],
+        "permalinkSelectors": [
+          "a[href*=\"/post/\"]"
+        ]
       }
     }
   ]

src/content/definitions/sites/mastodon.json

@@ -0,0 +1,13 @@
+{
+  "id": "mastodon",
+  "name": "Mastodon (official web interface)",
+  "urlPatterns": ["<all_urls>"],
+  "requiredSelectors": [".status__content", ".status, .detailed-status"],
+  "post": {
+    "rootSelectors": [".status", ".detailed-status"],
+    "textSelectors": [".status__content", ".e-content"],
+    "excludedSelectors": [".status__content__spoiler-link", ".status__content__read-more-button"],
+    "idAttributes": ["data-id"],
+    "permalinkSelectors": ["a.status__relative-time[href]", "a.detailed-status__datetime[href]", "a.u-url.u-uid[href]"]
+  }
+}

src/content/definitions/sites/threads.json

@@ -0,0 +1,17 @@
+{
+  "id": "threads",
+  "name": "Threads",
+  "urlPatterns": [
+    "https://threads.com/*",
+    "https://www.threads.com/*",
+    "https://threads.net/*",
+    "https://www.threads.net/*"
+  ],
+  "requiredSelectors": ["[role=\"article\"], article, [data-pressable-container=\"true\"], div[data-pagelet^=\"threads_post_page_\"]"],
+  "post": {
+    "rootSelectors": ["[role=\"article\"]", "article", "[data-pressable-container=\"true\"]", "div[data-pagelet^=\"threads_post_page_\"]"],
+    "textSelectors": ["span[dir=\"auto\"]"],
+    "excludedSelectors": ["button", "[role=\"button\"]", "time", "svg", "a span", "time span", "[role=\"button\"] span"],
+    "permalinkSelectors": ["a[href*=\"/post/\"]"]
+  }
+}

tests/definition-engine.test.ts

@@ -97,7 +97,7 @@ describe("DefinitionEngine", () => {
 
   it("selects Threads on current and legacy domains", () => {
     const threads = BUILTIN_DEFINITIONS.find((candidate) => candidate.id === "threads")!;
-    document.body.innerHTML = '<div role="article" data-pressable-container="true"></div>';
+    document.body.innerHTML = '<div data-pressable-container="true"></div>';
     for (const url of ["https://threads.com/home", "https://www.threads.com/@alice/post/ABC123", "https://threads.net/home", "https://www.threads.net/@alice/post/ABC123"]) {
       expect(selectDefinition([threads], url)).toBe(threads);
     }

tests/fixtures/threads-feed.html

@@ -1,5 +1,5 @@
 <main>
-  <div role="article" data-pressable-container="true">
+  <div data-pressable-container="true">
     <a href="/@alice"><span dir="auto">alice</span></a>
     <a href="/@alice/post/ABC123"><time><span dir="auto">12h</span></time></a>
     <span dir="auto">First Threads post with <span>visible text</span></span>

tools/generate-definitions.mjs

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+import { readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const repositoryRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+const definitionsDir = join(repositoryRoot, "src", "content", "definitions");
+const sitesDir = join(definitionsDir, "sites");
+const outputFile = join(definitionsDir, "default.json");
+const checkOnly = process.argv.includes("--check");
+
+function readJson(file) {
+  try {
+    return JSON.parse(readFileSync(file, "utf8"));
+  } catch (error) {
+    throw new Error(`Cannot read valid JSON from ${file}: ${error.message}`);
+  }
+}
+
+function requireString(value, field, file) {
+  if (typeof value !== "string" || !value.trim()) throw new Error(`${file}: ${field} must be a non-empty string.`);
+}
+
+function validateDefinition(definition, file) {
+  if (!definition || typeof definition !== "object" || Array.isArray(definition)) throw new Error(`${file}: a site definition must be an object.`);
+  requireString(definition.id, "id", file);
+  requireString(definition.name, "name", file);
+  if (!Array.isArray(definition.urlPatterns) || definition.urlPatterns.length === 0) throw new Error(`${file}: urlPatterns must be a non-empty array.`);
+  if (!Array.isArray(definition.requiredSelectors) || definition.requiredSelectors.length === 0) throw new Error(`${file}: requiredSelectors must be a non-empty array.`);
+  if (!definition.post || typeof definition.post !== "object" || Array.isArray(definition.post)) throw new Error(`${file}: post must be an object.`);
+}
+
+const collection = readJson(join(definitionsDir, "collection.json"));
+if (!collection || typeof collection !== "object" || Array.isArray(collection)) throw new Error("collection.json must be an object.");
+if (collection.schemaVersion !== 1) throw new Error("collection.json: schemaVersion must be 1.");
+requireString(collection.collectionId, "collectionId", "collection.json");
+requireString(collection.name, "name", "collection.json");
+
+const definitions = readdirSync(sitesDir)
+  .filter((file) => file.endsWith(".json"))
+  .sort()
+  .map((file) => {
+    const definition = readJson(join(sitesDir, file));
+    validateDefinition(definition, join("sites", file));
+    return definition;
+  });
+
+if (definitions.length === 0) throw new Error("No site definition JSON files were found.");
+
+const ids = new Set();
+for (const definition of definitions) {
+  if (ids.has(definition.id)) throw new Error(`Duplicate definition ID: ${definition.id}.`);
+  ids.add(definition.id);
+}
+
+const generated = `${JSON.stringify({ ...collection, definitions }, null, 2)}\n`;
+if (checkOnly) {
+  let current;
+  try { current = readFileSync(outputFile, "utf8"); } catch { throw new Error(`${basename(outputFile)} is missing. Run npm run definitions:generate.`); }
+  if (current !== generated) throw new Error(`${basename(outputFile)} is out of date. Run npm run definitions:generate and commit the result.`);
+  console.log("Definition collection is up to date.");
+} else {
+  writeFileSync(outputFile, generated);
+  console.log(`Generated ${basename(outputFile)} from ${definitions.length} site definition(s).`);
+}