Update run-reports to generate per-domain reports

README.md

@@ -22,6 +22,18 @@ python scripts/generate_reports.py weekly
 python scripts/generate_reports.py monthly
 ```
 
+Each command accepts optional flags to generate per-domain reports. Use
+`--domain <name>` to limit output to a specific domain or `--all-domains`
+to generate a subdirectory for every domain found in the database:
+
+```bash
+# Hourly reports for example.com only
+python scripts/generate_reports.py hourly --domain example.com
+
+# Weekly reports for all domains individually
+python scripts/generate_reports.py weekly --all-domains
+```
+
 Reports are written under the `output/` directory. Each command updates the corresponding `<interval>.json` file and produces an HTML dashboard using Chart.js.
 
 ### Configuring Reports
@@ -73,7 +85,7 @@ Use the `run-reports.sh` script to run all report intervals in one step. The scr
 ./run-reports.sh
 ```
 
-Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`.
+Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/<domain>/<interval>` alongside the aggregate data.
 
 ## Serving Reports with Nginx
 

run-reports.sh

@@ -18,13 +18,20 @@ else
     source .venv/bin/activate
 fi
 
-# Generate all reports
-echo "[INFO] Generating reports..."
+# Generate reports for all domains combined
+echo "[INFO] Generating aggregate reports..."
 python scripts/generate_reports.py hourly
 python scripts/generate_reports.py daily
 python scripts/generate_reports.py weekly
 python scripts/generate_reports.py monthly
 
+# Generate reports for each individual domain
+echo "[INFO] Generating per-domain reports..."
+python scripts/generate_reports.py hourly --all-domains
+python scripts/generate_reports.py daily --all-domains
+python scripts/generate_reports.py weekly --all-domains
+python scripts/generate_reports.py monthly --all-domains
+
 # Deactivate to keep cron environment clean
 if type deactivate >/dev/null 2>&1; then
     deactivate

scripts/generate_reports.py

@@ -1,7 +1,7 @@
 import json
 import sqlite3
 from pathlib import Path
-from typing import List, Dict
+from typing import List, Dict, Optional
 
 import yaml
 
@@ -26,6 +26,16 @@ INTERVAL_FORMATS = {
 
 app = typer.Typer(help="Generate aggregated log reports")
 
+
+def _get_domains() -> List[str]:
+    """Return a sorted list of unique domains from the logs table."""
+    conn = sqlite3.connect(DB_PATH)
+    cur = conn.cursor()
+    cur.execute("SELECT DISTINCT host FROM logs ORDER BY host")
+    domains = [row[0] for row in cur.fetchall()]
+    conn.close()
+    return domains
+
 def _load_config() -> List[Dict]:
     if not REPORT_CONFIG.exists():
         typer.echo(f"Config file not found: {REPORT_CONFIG}")
@@ -55,7 +65,7 @@ def _bucket_expr(interval: str) -> str:
         raise typer.Exit(1)
     return f"strftime('{fmt}', datetime(time))"
 
-def _generate_interval(interval: str) -> None:
+def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
     cfg = _load_config()
     if not cfg:
         typer.echo("No report definitions found")
@@ -66,13 +76,25 @@ def _generate_interval(interval: str) -> None:
     conn = sqlite3.connect(DB_PATH)
     cur = conn.cursor()
 
+    # Create a temporary view so queries can easily be filtered by domain
+    cur.execute("DROP VIEW IF EXISTS logs_view")
+    if domain:
+        cur.execute(
+            "CREATE TEMP VIEW logs_view AS SELECT * FROM logs WHERE host = ?",
+            (domain,),
+        )
+        out_dir = OUTPUT_DIR / domain / interval
+    else:
+        cur.execute("CREATE TEMP VIEW logs_view AS SELECT * FROM logs")
         out_dir = OUTPUT_DIR / interval
+
     out_dir.mkdir(parents=True, exist_ok=True)
 
     report_list = []
     for definition in cfg:
         name = definition["name"]
         query = definition["query"].replace("{bucket}", bucket)
+        query = query.replace("FROM logs", "FROM logs_view")
         cur.execute(query)
         rows = cur.fetchall()
         headers = [c[0] for c in cur.description]
@@ -90,25 +112,71 @@ def _generate_interval(interval: str) -> None:
     _render_html(interval, report_list, out_dir / "index.html")
     typer.echo(f"Generated {interval} reports")
 
+
+def _generate_all_domains(interval: str) -> None:
+    """Generate reports for each unique domain."""
+    for domain in _get_domains():
+        _generate_interval(interval, domain)
+
 @app.command()
-def hourly() -> None:
+def hourly(
+    domain: Optional[str] = typer.Option(
+        None, help="Generate reports for a specific domain"
+    ),
+    all_domains: bool = typer.Option(
+        False, "--all-domains", help="Generate reports for each domain"
+    ),
+) -> None:
     """Generate hourly reports."""
-    _generate_interval("hourly")
+    if all_domains:
+        _generate_all_domains("hourly")
+    else:
+        _generate_interval("hourly", domain)
 
 @app.command()
-def daily() -> None:
+def daily(
+    domain: Optional[str] = typer.Option(
+        None, help="Generate reports for a specific domain"
+    ),
+    all_domains: bool = typer.Option(
+        False, "--all-domains", help="Generate reports for each domain"
+    ),
+) -> None:
     """Generate daily reports."""
-    _generate_interval("daily")
+    if all_domains:
+        _generate_all_domains("daily")
+    else:
+        _generate_interval("daily", domain)
 
 @app.command()
-def weekly() -> None:
+def weekly(
+    domain: Optional[str] = typer.Option(
+        None, help="Generate reports for a specific domain"
+    ),
+    all_domains: bool = typer.Option(
+        False, "--all-domains", help="Generate reports for each domain"
+    ),
+) -> None:
     """Generate weekly reports."""
-    _generate_interval("weekly")
+    if all_domains:
+        _generate_all_domains("weekly")
+    else:
+        _generate_interval("weekly", domain)
 
 @app.command()
-def monthly() -> None:
+def monthly(
+    domain: Optional[str] = typer.Option(
+        None, help="Generate reports for a specific domain"
+    ),
+    all_domains: bool = typer.Option(
+        False, "--all-domains", help="Generate reports for each domain"
+    ),
+) -> None:
     """Generate monthly reports."""
-    _generate_interval("monthly")
+    if all_domains:
+        _generate_all_domains("monthly")
+    else:
+        _generate_interval("monthly", domain)
 
 if __name__ == "__main__":
     app()