From 27a73ec4e8600344299d8c82d179b3b863800bf4 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 18 Jul 2025 02:10:46 -0500 Subject: [PATCH] Update run-reports to generate per-domain reports --- README.md | 14 +++++- run-reports.sh | 11 ++++- scripts/generate_reports.py | 90 ++++++++++++++++++++++++++++++++----- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 44e3058..21265a0 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,18 @@ python scripts/generate_reports.py weekly python scripts/generate_reports.py monthly ``` +Each command accepts optional flags to generate per-domain reports. Use +`--domain ` to limit output to a specific domain or `--all-domains` +to generate a subdirectory for every domain found in the database: + +```bash +# Hourly reports for example.com only +python scripts/generate_reports.py hourly --domain example.com + +# Weekly reports for all domains individually +python scripts/generate_reports.py weekly --all-domains +``` + Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and produces an HTML dashboard using Chart.js. ### Configuring Reports @@ -73,7 +85,7 @@ Use the `run-reports.sh` script to run all report intervals in one step. The scr ./run-reports.sh ``` -Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. +Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output//` alongside the aggregate data. ## Serving Reports with Nginx diff --git a/run-reports.sh b/run-reports.sh index 56f4c31..6a666be 100755 --- a/run-reports.sh +++ b/run-reports.sh @@ -18,13 +18,20 @@ else source .venv/bin/activate fi -# Generate all reports -echo "[INFO] Generating reports..." +# Generate reports for all domains combined +echo "[INFO] Generating aggregate reports..." python scripts/generate_reports.py hourly python scripts/generate_reports.py daily python scripts/generate_reports.py weekly python scripts/generate_reports.py monthly +# Generate reports for each individual domain +echo "[INFO] Generating per-domain reports..." +python scripts/generate_reports.py hourly --all-domains +python scripts/generate_reports.py daily --all-domains +python scripts/generate_reports.py weekly --all-domains +python scripts/generate_reports.py monthly --all-domains + # Deactivate to keep cron environment clean if type deactivate >/dev/null 2>&1; then deactivate diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index 890c947..f099d17 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -1,7 +1,7 @@ import json import sqlite3 from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Optional import yaml @@ -26,6 +26,16 @@ INTERVAL_FORMATS = { app = typer.Typer(help="Generate aggregated log reports") + +def _get_domains() -> List[str]: + """Return a sorted list of unique domains from the logs table.""" + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + cur.execute("SELECT DISTINCT host FROM logs ORDER BY host") + domains = [row[0] for row in cur.fetchall()] + conn.close() + return domains + def _load_config() -> List[Dict]: if not REPORT_CONFIG.exists(): typer.echo(f"Config file not found: {REPORT_CONFIG}") @@ -55,7 +65,7 @@ def _bucket_expr(interval: str) -> str: raise typer.Exit(1) return f"strftime('{fmt}', datetime(time))" -def _generate_interval(interval: str) -> None: +def _generate_interval(interval: str, domain: Optional[str] = None) -> None: cfg = _load_config() if not cfg: typer.echo("No report definitions found") @@ -66,13 +76,25 @@ def _generate_interval(interval: str) -> None: conn = sqlite3.connect(DB_PATH) cur = conn.cursor() - out_dir = OUTPUT_DIR / interval + # Create a temporary view so queries can easily be filtered by domain + cur.execute("DROP VIEW IF EXISTS logs_view") + if domain: + cur.execute( + "CREATE TEMP VIEW logs_view AS SELECT * FROM logs WHERE host = ?", + (domain,), + ) + out_dir = OUTPUT_DIR / domain / interval + else: + cur.execute("CREATE TEMP VIEW logs_view AS SELECT * FROM logs") + out_dir = OUTPUT_DIR / interval + out_dir.mkdir(parents=True, exist_ok=True) report_list = [] for definition in cfg: name = definition["name"] query = definition["query"].replace("{bucket}", bucket) + query = query.replace("FROM logs", "FROM logs_view") cur.execute(query) rows = cur.fetchall() headers = [c[0] for c in cur.description] @@ -90,25 +112,71 @@ def _generate_interval(interval: str) -> None: _render_html(interval, report_list, out_dir / "index.html") typer.echo(f"Generated {interval} reports") + +def _generate_all_domains(interval: str) -> None: + """Generate reports for each unique domain.""" + for domain in _get_domains(): + _generate_interval(interval, domain) + @app.command() -def hourly() -> None: +def hourly( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: """Generate hourly reports.""" - _generate_interval("hourly") + if all_domains: + _generate_all_domains("hourly") + else: + _generate_interval("hourly", domain) @app.command() -def daily() -> None: +def daily( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: """Generate daily reports.""" - _generate_interval("daily") + if all_domains: + _generate_all_domains("daily") + else: + _generate_interval("daily", domain) @app.command() -def weekly() -> None: +def weekly( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: """Generate weekly reports.""" - _generate_interval("weekly") + if all_domains: + _generate_all_domains("weekly") + else: + _generate_interval("weekly", domain) @app.command() -def monthly() -> None: +def monthly( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: """Generate monthly reports.""" - _generate_interval("monthly") + if all_domains: + _generate_all_domains("monthly") + else: + _generate_interval("monthly", domain) if __name__ == "__main__": app()