From fc07dae5499cfddf97b7478428998374bbe85825 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 18 Jul 2025 22:29:52 -0500 Subject: [PATCH 01/48] Add DataTables table chart type --- AGENTS.md | 6 ++++++ reports.yml | 14 +++++++------- templates/report.html | 39 ++++++++++++++++++++++++++++++++------- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 52210c6..4cdfa62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,12 @@ This document outlines general practices and expectations for AI agents assistin * Use latest CDN version for embedded dashboards * Charts should be rendered from pre-generated JSON blobs in `/json/` +### Tables: DataTables + +* Use DataTables via CDN for reports with `chart: table` +* Requires jQuery from a CDN +* Table data comes from the same `/json/` files as charts + ### Styling: Bulma CSS * Use via CDN or vendored minified copy (to keep reports fully static) diff --git a/reports.yml b/reports.yml index 754d311..01716e1 100644 --- a/reports.yml +++ b/reports.yml @@ -1,6 +1,6 @@ - name: hits label: Hits - chart: bar + chart: line query: | SELECT {bucket} AS bucket, COUNT(*) AS value @@ -20,7 +20,7 @@ - name: cache_status_breakdown label: Cache Status - chart: bar + chart: polarArea query: | SELECT cache_status AS bucket, COUNT(*) AS value @@ -30,7 +30,7 @@ - name: domain_traffic label: Top Domains - chart: bar + chart: table query: | SELECT host AS bucket, COUNT(*) AS value @@ -50,7 +50,7 @@ - name: top_paths label: Top Paths - chart: bar + chart: table query: | SELECT path AS bucket, COUNT(*) AS value @@ -65,7 +65,7 @@ - name: user_agents label: User Agents - chart: bar + chart: table query: | SELECT user_agent AS bucket, COUNT(*) AS value @@ -76,7 +76,7 @@ - name: referrers label: Referrers - chart: bar + chart: table query: | SELECT referer AS bucket, COUNT(*) AS value @@ -87,7 +87,7 @@ - name: status_distribution label: HTTP Statuses - chart: bar + chart: stackedBar query: | SELECT CASE WHEN status BETWEEN 200 AND 299 THEN '2xx' diff --git a/templates/report.html b/templates/report.html index 3288269..bbb6e37 100644 --- a/templates/report.html +++ b/templates/report.html @@ -4,7 +4,10 @@ {{ interval.title() }} Report + + +
@@ -12,7 +15,11 @@ {% for report in reports %}

{{ report.label }}

+ {% if report.chart == 'table' %} +
+ {% else %} + {% endif %}
{% endfor %}
@@ -22,10 +29,32 @@ fetch(rep.json) .then(r => r.json()) .then(data => { + if (rep.chart === 'table') { + const rows = data.map(x => [x.bucket, x.value]); + new DataTable('#table-' + rep.name, { + data: rows, + columns: [ + { title: 'Bucket' }, + { title: 'Value' } + ] + }); + return; + } + const labels = data.map(x => x.bucket); const values = data.map(x => x.value); + const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart; + const options = { + scales: { + y: { beginAtZero: true } + } + }; + if (rep.chart === 'stackedBar') { + options.scales.x = { stacked: true }; + options.scales.y.stacked = true; + } new Chart(document.getElementById('chart-' + rep.name), { - type: rep.chart, + type: chartType, data: { labels: labels, datasets: [{ @@ -34,14 +63,10 @@ backgroundColor: 'rgba(54, 162, 235, 0.5)', borderColor: 'rgba(54, 162, 235, 1)', borderWidth: 1, - fill: rep.chart !== 'bar', + fill: rep.chart !== 'bar' && rep.chart !== 'stackedBar', }] }, - options: { - scales: { - y: { beginAtZero: true } - } - } + options: options }); }); }); From 44ee039ca6f9a75b4cec419958f2438c13a5d065 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 18 Jul 2025 23:03:04 -0500 Subject: [PATCH 02/48] Add color support for charts --- reports.yml | 14 +++++++++++++- scripts/generate_reports.py | 19 +++++++++++-------- templates/report.html | 25 +++++++++++++++++-------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/reports.yml b/reports.yml index 01716e1..0df9913 100644 --- a/reports.yml +++ b/reports.yml @@ -27,6 +27,13 @@ FROM logs GROUP BY cache_status ORDER BY value DESC + colors: + - "#3273dc" + - "#23d160" + - "#ffdd57" + - "#ff3860" + - "#7957d5" + - "#363636" - name: domain_traffic label: Top Domains @@ -87,7 +94,7 @@ - name: status_distribution label: HTTP Statuses - chart: stackedBar + chart: pie query: | SELECT CASE WHEN status BETWEEN 200 AND 299 THEN '2xx' @@ -99,3 +106,8 @@ FROM logs GROUP BY bucket ORDER BY bucket + colors: + - "#48c78e" + - "#209cee" + - "#ffdd57" + - "#f14668" diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index 6118978..249506d 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -108,14 +108,17 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: data = [dict(zip(headers, row)) for row in rows] json_path = out_dir / f"{name}.json" _save_json(json_path, data) - report_list.append( - { - "name": name, - "label": definition.get("label", name.title()), - "chart": definition.get("chart", "line"), - "json": f"{name}.json", - } - ) + entry = { + "name": name, + "label": definition.get("label", name.title()), + "chart": definition.get("chart", "line"), + "json": f"{name}.json", + } + if "color" in definition: + entry["color"] = definition["color"] + if "colors" in definition: + entry["colors"] = definition["colors"] + report_list.append(entry) _save_json(out_dir / "reports.json", report_list) _render_html(interval, report_list, out_dir / "index.html") diff --git a/templates/report.html b/templates/report.html index bbb6e37..6e1dd37 100644 --- a/templates/report.html +++ b/templates/report.html @@ -53,18 +53,27 @@ options.scales.x = { stacked: true }; options.scales.y.stacked = true; } + const dataset = { + label: rep.label, + data: values, + borderWidth: 1, + fill: rep.chart !== 'bar' && rep.chart !== 'stackedBar' + }; + if (rep.colors) { + dataset.backgroundColor = rep.colors; + dataset.borderColor = rep.colors; + } else if (rep.color) { + dataset.backgroundColor = rep.color; + dataset.borderColor = rep.color; + } else { + dataset.backgroundColor = 'rgba(54, 162, 235, 0.5)'; + dataset.borderColor = 'rgba(54, 162, 235, 1)'; + } new Chart(document.getElementById('chart-' + rep.name), { type: chartType, data: { labels: labels, - datasets: [{ - label: rep.label, - data: values, - backgroundColor: 'rgba(54, 162, 235, 0.5)', - borderColor: 'rgba(54, 162, 235, 1)', - borderWidth: 1, - fill: rep.chart !== 'bar' && rep.chart !== 'stackedBar', - }] + datasets: [dataset] }, options: options }); From ab2af1015a2f10c75b277f84c22fa32ca0c0bfbc Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Fri, 18 Jul 2025 23:20:13 -0500 Subject: [PATCH 03/48] Switch to snippet-based reports --- README.md | 10 ++-- scripts/generate_reports.py | 11 +++-- templates/index.html | 87 +++++++++++++++++++++++++++++------ templates/report.html | 84 --------------------------------- templates/report_snippet.html | 8 ++++ tests/test_reports.py | 3 ++ 6 files changed, 97 insertions(+), 106 deletions(-) delete mode 100644 templates/report.html create mode 100644 templates/report_snippet.html diff --git a/README.md b/README.md index be7155f..d206658 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Per-domain Nginx log analytics with hybrid static reports and live insights. ## Generating Reports -Use the `generate_reports.py` script to build aggregated JSON and HTML files from `database/ngxstat.db`. +Use the `generate_reports.py` script to build aggregated JSON and HTML snippet files from `database/ngxstat.db`. Create a virtual environment and install dependencies: @@ -34,7 +34,7 @@ python scripts/generate_reports.py hourly --domain example.com python scripts/generate_reports.py weekly --all-domains ``` -Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and produces an HTML dashboard using Chart.js. +Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and writes one HTML snippet per report. These snippets are loaded dynamically by the main dashboard using Chart.js and DataTables. ### Configuring Reports @@ -44,8 +44,8 @@ and `value` columns. The special token `{bucket}` is replaced with the appropriate SQLite `strftime` expression for each interval (hourly, daily, weekly or monthly) so that a single definition works across all durations. When `generate_reports.py` runs, every definition is executed for the requested -interval and creates `output//.json` along with an HTML -dashboard. +interval and creates `output//.json` plus a small HTML snippet +`output//.html` used by the dashboard. Example snippet: @@ -85,7 +85,7 @@ Use the `run-reports.sh` script to run all report intervals in one step. The scr ./run-reports.sh ``` -Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/domains//` alongside the aggregate data. +Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/domains//` alongside the aggregate data. After generation, open `output/index.html` in your browser to browse the reports. ## Serving Reports with Nginx diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index 249506d..8b9fad6 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -54,10 +54,12 @@ def _save_json(path: Path, data: List[Dict]) -> None: path.write_text(json.dumps(data, indent=2)) -def _render_html(interval: str, reports: List[Dict], out_path: Path) -> None: +def _render_snippet(report: Dict, out_dir: Path) -> None: + """Render a single report snippet to ``.html`` inside ``out_dir``.""" env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) - template = env.get_template("report.html") - out_path.write_text(template.render(interval=interval, reports=reports)) + template = env.get_template("report_snippet.html") + snippet_path = out_dir / f"{report['name']}.html" + snippet_path.write_text(template.render(report=report)) def _bucket_expr(interval: str) -> str: @@ -113,15 +115,16 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: "label": definition.get("label", name.title()), "chart": definition.get("chart", "line"), "json": f"{name}.json", + "html": f"{name}.html", } if "color" in definition: entry["color"] = definition["color"] if "colors" in definition: entry["colors"] = definition["colors"] + _render_snippet(entry, out_dir) report_list.append(entry) _save_json(out_dir / "reports.json", report_list) - _render_html(interval, report_list, out_dir / "index.html") typer.echo(f"Generated {interval} reports") diff --git a/templates/index.html b/templates/index.html index 222e347..08b5732 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,6 +4,7 @@ ngxstat Reports +
@@ -31,42 +32,102 @@
- +
+ + + diff --git a/templates/report.html b/templates/report.html deleted file mode 100644 index 6e1dd37..0000000 --- a/templates/report.html +++ /dev/null @@ -1,84 +0,0 @@ - - - - - {{ interval.title() }} Report - - - - - - - -
-

{{ interval.title() }} Report

- {% for report in reports %} -
-

{{ report.label }}

- {% if report.chart == 'table' %} -
- {% else %} - - {% endif %} -
- {% endfor %} -
- - - diff --git a/templates/report_snippet.html b/templates/report_snippet.html new file mode 100644 index 0000000..c0a69d6 --- /dev/null +++ b/templates/report_snippet.html @@ -0,0 +1,8 @@ +
+

{{ report.label }}

+ {% if report.chart == 'table' %} +
+ {% else %} + + {% endif %} +
diff --git a/tests/test_reports.py b/tests/test_reports.py index dfd2c6d..f1be1d0 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -106,6 +106,9 @@ def test_generate_interval(tmp_path, sample_reports, monkeypatch): assert error_rate[0]["value"] == pytest.approx(50.0) reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text()) assert {r["name"] for r in reports} == {"hits", "error_rate"} + for r in reports: + snippet = tmp_path / "output" / "hourly" / r["html"] + assert snippet.exists() def test_generate_interval_domain_filter(tmp_path, sample_reports, monkeypatch): From a3f06fd9e2b979699f13ba97fa00d108d07e3b08 Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Sat, 19 Jul 2025 00:09:26 -0500 Subject: [PATCH 04/48] Add global report generation --- run-reports.sh | 1 + scripts/generate_reports.py | 54 +++++++++++++++++++++++++++++++++++++ tests/test_reports.py | 28 +++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/run-reports.sh b/run-reports.sh index ade36c6..a0d718f 100755 --- a/run-reports.sh +++ b/run-reports.sh @@ -24,6 +24,7 @@ python scripts/generate_reports.py hourly python scripts/generate_reports.py daily python scripts/generate_reports.py weekly python scripts/generate_reports.py monthly +python scripts/generate_reports.py global # Generate reports for each individual domain echo "[INFO] Generating per-domain reports..." diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index 8b9fad6..e7c42cb 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -101,6 +101,10 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: report_list = [] for definition in cfg: + if "{bucket}" not in definition["query"] or definition.get("global"): + # Global reports are generated separately + continue + name = definition["name"] query = definition["query"].replace("{bucket}", bucket) query = query.replace("FROM logs", "FROM logs_view") @@ -154,6 +158,50 @@ def _generate_root_index() -> None: typer.echo(f"Generated root index at {out_path}") +def _generate_global() -> None: + """Generate reports that do not depend on an interval.""" + cfg = _load_config() + if not cfg: + typer.echo("No report definitions found") + return + + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + out_dir = OUTPUT_DIR / "global" + out_dir.mkdir(parents=True, exist_ok=True) + + report_list = [] + for definition in cfg: + if "{bucket}" in definition["query"] and not definition.get("global"): + continue + + name = definition["name"] + query = definition["query"] + cur.execute(query) + rows = cur.fetchall() + headers = [c[0] for c in cur.description] + data = [dict(zip(headers, row)) for row in rows] + json_path = out_dir / f"{name}.json" + _save_json(json_path, data) + entry = { + "name": name, + "label": definition.get("label", name.title()), + "chart": definition.get("chart", "line"), + "json": f"{name}.json", + "html": f"{name}.html", + } + if "color" in definition: + entry["color"] = definition["color"] + if "colors" in definition: + entry["colors"] = definition["colors"] + _render_snippet(entry, out_dir) + report_list.append(entry) + + _save_json(out_dir / "reports.json", report_list) + typer.echo("Generated global reports") + + @app.command() def hourly( domain: Optional[str] = typer.Option( @@ -218,6 +266,12 @@ def monthly( _generate_interval("monthly", domain) +@app.command("global") +def global_reports() -> None: + """Generate global reports.""" + _generate_global() + + @app.command() def index() -> None: """Generate the root index page linking all reports.""" diff --git a/tests/test_reports.py b/tests/test_reports.py index f1be1d0..d8259f2 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -80,6 +80,14 @@ def sample_reports(tmp_path): FROM logs GROUP BY bucket ORDER BY bucket +- name: domain_totals + global: true + query: | + SELECT host AS bucket, + COUNT(*) AS value + FROM logs + GROUP BY host + ORDER BY value DESC """ ) return cfg @@ -161,3 +169,23 @@ def test_generate_root_index(tmp_path, sample_reports, monkeypatch): # check for domain options assert '