From ab2af1015a2f10c75b277f84c22fa32ca0c0bfbc Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 18 Jul 2025 23:20:13 -0500 Subject: [PATCH 01/46] Switch to snippet-based reports --- README.md | 10 ++-- scripts/generate_reports.py | 11 +++-- templates/index.html | 87 +++++++++++++++++++++++++++++------ templates/report.html | 84 --------------------------------- templates/report_snippet.html | 8 ++++ tests/test_reports.py | 3 ++ 6 files changed, 97 insertions(+), 106 deletions(-) delete mode 100644 templates/report.html create mode 100644 templates/report_snippet.html diff --git a/README.md b/README.md index be7155f..d206658 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Per-domain Nginx log analytics with hybrid static reports and live insights. ## Generating Reports -Use the `generate_reports.py` script to build aggregated JSON and HTML files from `database/ngxstat.db`. +Use the `generate_reports.py` script to build aggregated JSON and HTML snippet files from `database/ngxstat.db`. Create a virtual environment and install dependencies: @@ -34,7 +34,7 @@ python scripts/generate_reports.py hourly --domain example.com python scripts/generate_reports.py weekly --all-domains ``` -Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and produces an HTML dashboard using Chart.js. +Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and writes one HTML snippet per report. These snippets are loaded dynamically by the main dashboard using Chart.js and DataTables. ### Configuring Reports @@ -44,8 +44,8 @@ and `value` columns. The special token `{bucket}` is replaced with the appropriate SQLite `strftime` expression for each interval (hourly, daily, weekly or monthly) so that a single definition works across all durations. When `generate_reports.py` runs, every definition is executed for the requested -interval and creates `output//.json` along with an HTML -dashboard. +interval and creates `output//.json` plus a small HTML snippet +`output//.html` used by the dashboard. Example snippet: @@ -85,7 +85,7 @@ Use the `run-reports.sh` script to run all report intervals in one step. The scr ./run-reports.sh ``` -Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/domains//` alongside the aggregate data. +Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/domains//` alongside the aggregate data. After generation, open `output/index.html` in your browser to browse the reports. ## Serving Reports with Nginx diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index 249506d..8b9fad6 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -54,10 +54,12 @@ def _save_json(path: Path, data: List[Dict]) -> None: path.write_text(json.dumps(data, indent=2)) -def _render_html(interval: str, reports: List[Dict], out_path: Path) -> None: +def _render_snippet(report: Dict, out_dir: Path) -> None: + """Render a single report snippet to ``.html`` inside ``out_dir``.""" env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) - template = env.get_template("report.html") - out_path.write_text(template.render(interval=interval, reports=reports)) + template = env.get_template("report_snippet.html") + snippet_path = out_dir / f"{report['name']}.html" + snippet_path.write_text(template.render(report=report)) def _bucket_expr(interval: str) -> str: @@ -113,15 +115,16 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: "label": definition.get("label", name.title()), "chart": definition.get("chart", "line"), "json": f"{name}.json", + "html": f"{name}.html", } if "color" in definition: entry["color"] = definition["color"] if "colors" in definition: entry["colors"] = definition["colors"] + _render_snippet(entry, out_dir) report_list.append(entry) _save_json(out_dir / "reports.json", report_list) - _render_html(interval, report_list, out_dir / "index.html") typer.echo(f"Generated {interval} reports") diff --git a/templates/index.html b/templates/index.html index 222e347..08b5732 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,6 +4,7 @@ ngxstat Reports +
@@ -31,42 +32,102 @@
- +
+ + + diff --git a/templates/report.html b/templates/report.html deleted file mode 100644 index 6e1dd37..0000000 --- a/templates/report.html +++ /dev/null @@ -1,84 +0,0 @@ - - - - - {{ interval.title() }} Report - - - - - - - -
-

{{ interval.title() }} Report

- {% for report in reports %} -
-

{{ report.label }}

- {% if report.chart == 'table' %} -
- {% else %} - - {% endif %} -
- {% endfor %} -
- - - diff --git a/templates/report_snippet.html b/templates/report_snippet.html new file mode 100644 index 0000000..c0a69d6 --- /dev/null +++ b/templates/report_snippet.html @@ -0,0 +1,8 @@ +
+

{{ report.label }}

+ {% if report.chart == 'table' %} +
+ {% else %} + + {% endif %} +
diff --git a/tests/test_reports.py b/tests/test_reports.py index dfd2c6d..f1be1d0 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -106,6 +106,9 @@ def test_generate_interval(tmp_path, sample_reports, monkeypatch): assert error_rate[0]["value"] == pytest.approx(50.0) reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text()) assert {r["name"] for r in reports} == {"hits", "error_rate"} + for r in reports: + snippet = tmp_path / "output" / "hourly" / r["html"] + assert snippet.exists() def test_generate_interval_domain_filter(tmp_path, sample_reports, monkeypatch): From a3f06fd9e2b979699f13ba97fa00d108d07e3b08 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 00:09:26 -0500 Subject: [PATCH 02/46] Add global report generation --- run-reports.sh | 1 + scripts/generate_reports.py | 54 +++++++++++++++++++++++++++++++++++++ tests/test_reports.py | 28 +++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/run-reports.sh b/run-reports.sh index ade36c6..a0d718f 100755 --- a/run-reports.sh +++ b/run-reports.sh @@ -24,6 +24,7 @@ python scripts/generate_reports.py hourly python scripts/generate_reports.py daily python scripts/generate_reports.py weekly python scripts/generate_reports.py monthly +python scripts/generate_reports.py global # Generate reports for each individual domain echo "[INFO] Generating per-domain reports..." diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index 8b9fad6..e7c42cb 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -101,6 +101,10 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: report_list = [] for definition in cfg: + if "{bucket}" not in definition["query"] or definition.get("global"): + # Global reports are generated separately + continue + name = definition["name"] query = definition["query"].replace("{bucket}", bucket) query = query.replace("FROM logs", "FROM logs_view") @@ -154,6 +158,50 @@ def _generate_root_index() -> None: typer.echo(f"Generated root index at {out_path}") +def _generate_global() -> None: + """Generate reports that do not depend on an interval.""" + cfg = _load_config() + if not cfg: + typer.echo("No report definitions found") + return + + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + out_dir = OUTPUT_DIR / "global" + out_dir.mkdir(parents=True, exist_ok=True) + + report_list = [] + for definition in cfg: + if "{bucket}" in definition["query"] and not definition.get("global"): + continue + + name = definition["name"] + query = definition["query"] + cur.execute(query) + rows = cur.fetchall() + headers = [c[0] for c in cur.description] + data = [dict(zip(headers, row)) for row in rows] + json_path = out_dir / f"{name}.json" + _save_json(json_path, data) + entry = { + "name": name, + "label": definition.get("label", name.title()), + "chart": definition.get("chart", "line"), + "json": f"{name}.json", + "html": f"{name}.html", + } + if "color" in definition: + entry["color"] = definition["color"] + if "colors" in definition: + entry["colors"] = definition["colors"] + _render_snippet(entry, out_dir) + report_list.append(entry) + + _save_json(out_dir / "reports.json", report_list) + typer.echo("Generated global reports") + + @app.command() def hourly( domain: Optional[str] = typer.Option( @@ -218,6 +266,12 @@ def monthly( _generate_interval("monthly", domain) +@app.command("global") +def global_reports() -> None: + """Generate global reports.""" + _generate_global() + + @app.command() def index() -> None: """Generate the root index page linking all reports.""" diff --git a/tests/test_reports.py b/tests/test_reports.py index f1be1d0..d8259f2 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -80,6 +80,14 @@ def sample_reports(tmp_path): FROM logs GROUP BY bucket ORDER BY bucket +- name: domain_totals + global: true + query: | + SELECT host AS bucket, + COUNT(*) AS value + FROM logs + GROUP BY host + ORDER BY value DESC """ ) return cfg @@ -161,3 +169,23 @@ def test_generate_root_index(tmp_path, sample_reports, monkeypatch): # check for domain options assert '