wagesj45/ngxstat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Switch to snippet-based reports

Browse source
This commit is contained in:
Jordan Wages 2025-07-18 23:20:13 -05:00
commit ab2af1015a
6 changed files with 97 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

10
README.md
View file

@ -3,7 +3,7 @@ Per-domain Nginx log analytics with hybrid static reports and live insights.
## Generating Reports ## Generating Reports
Use the `generate_reports.py` script to build aggregated JSON and HTML files from `database/ngxstat.db`. Use the `generate_reports.py` script to build aggregated JSON and HTML snippet files from `database/ngxstat.db`.
Create a virtual environment and install dependencies: Create a virtual environment and install dependencies:
@ -34,7 +34,7 @@ python scripts/generate_reports.py hourly --domain example.com
python scripts/generate_reports.py weekly --all-domains python scripts/generate_reports.py weekly --all-domains
``` ```
Reports are written under the `output/` directory. Each command updates the corresponding `<interval>.json` file and produces an HTML dashboard using Chart.js. Reports are written under the `output/` directory. Each command updates the corresponding `<interval>.json` file and writes one HTML snippet per report. These snippets are loaded dynamically by the main dashboard using Chart.js and DataTables.
### Configuring Reports ### Configuring Reports
@ -44,8 +44,8 @@ and `value` columns. The special token `{bucket}` is replaced with the
appropriate SQLite `strftime` expression for each interval (hourly, daily, appropriate SQLite `strftime` expression for each interval (hourly, daily,
weekly or monthly) so that a single definition works across all durations. weekly or monthly) so that a single definition works across all durations.
When `generate_reports.py` runs, every definition is executed for the requested When `generate_reports.py` runs, every definition is executed for the requested
interval and creates `output/<interval>/<name>.json` along with an HTML interval and creates `output/<interval>/<name>.json` plus a small HTML snippet
dashboard. `output/<interval>/<name>.html` used by the dashboard.
Example snippet: Example snippet:
@ -85,7 +85,7 @@ Use the `run-reports.sh` script to run all report intervals in one step. The scr
./run-reports.sh ./run-reports.sh
``` ```
Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/domains/<domain>/<interval>` alongside the aggregate data. Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. It also detects all unique domains found in the database and writes per-domain reports to `output/domains/<domain>/<interval>` alongside the aggregate data. After generation, open `output/index.html` in your browser to browse the reports.
## Serving Reports with Nginx ## Serving Reports with Nginx

11
scripts/generate_reports.py
View file

@ -54,10 +54,12 @@ def _save_json(path: Path, data: List[Dict]) -> None:
path.write_text(json.dumps(data, indent=2)) path.write_text(json.dumps(data, indent=2))
def _render_html(interval: str, reports: List[Dict], out_path: Path) -> None: def _render_snippet(report: Dict, out_dir: Path) -> None:
"""Render a single report snippet to ``<name>.html`` inside ``out_dir``."""
env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) env = Environment(loader=FileSystemLoader(TEMPLATE_DIR))
template = env.get_template("report.html") template = env.get_template("report_snippet.html")
out_path.write_text(template.render(interval=interval, reports=reports)) snippet_path = out_dir / f"{report['name']}.html"
snippet_path.write_text(template.render(report=report))
def _bucket_expr(interval: str) -> str: def _bucket_expr(interval: str) -> str:
@ -113,15 +115,16 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
"label": definition.get("label", name.title()), "label": definition.get("label", name.title()),
"chart": definition.get("chart", "line"), "chart": definition.get("chart", "line"),
"json": f"{name}.json", "json": f"{name}.json",
"html": f"{name}.html",
} }
if "color" in definition: if "color" in definition:
entry["color"] = definition["color"] entry["color"] = definition["color"]
if "colors" in definition: if "colors" in definition:
entry["colors"] = definition["colors"] entry["colors"] = definition["colors"]
_render_snippet(entry, out_dir)
report_list.append(entry) report_list.append(entry)
_save_json(out_dir / "reports.json", report_list) _save_json(out_dir / "reports.json", report_list)
_render_html(interval, report_list, out_dir / "index.html")
typer.echo(f"Generated {interval} reports") typer.echo(f"Generated {interval} reports")

87
templates/index.html
View file

@ -4,6 +4,7 @@
<meta charset="utf-8"> <meta charset="utf-8">
<title>ngxstat Reports</title> <title>ngxstat Reports</title>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css"> <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
<link rel="stylesheet" href="https://cdn.datatables.net/1.13.4/css/jquery.dataTables.min.css">
</head> </head>
<body class="section"> <body class="section">
<div class="container"> <div class="container">
@ -31,42 +32,102 @@
<span class="icon is-small is-left"><i data-feather="server"></i></span> <span class="icon is-small is-left"><i data-feather="server"></i></span>
</div> </div>
</div> </div>
<iframe id="report-frame" src="" style="width:100%;border:none;"></iframe> <div id="reports-container"></div>
</div> </div>
<script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/jquery@3.7.0/dist/jquery.min.js"></script>
<script src="https://cdn.datatables.net/1.13.4/js/jquery.dataTables.min.js"></script>
<script> <script>
const intervalSelect = document.getElementById('interval-select'); const intervalSelect = document.getElementById('interval-select');
const domainSelect = document.getElementById('domain-select'); const domainSelect = document.getElementById('domain-select');
const iframe = document.getElementById('report-frame'); const container = document.getElementById('reports-container');
let currentInterval = intervalSelect.value; let currentInterval = intervalSelect.value;
let currentDomain = domainSelect.value; let currentDomain = domainSelect.value;
function updateFrame() { function initReport(rep, base) {
fetch(base + '/' + rep.json)
.then(r => r.json())
.then(data => {
if (rep.chart === 'table') {
const rows = data.map(x => [x.bucket, x.value]);
new DataTable('#table-' + rep.name, {
data: rows,
columns: [
{ title: 'Bucket' },
{ title: 'Value' }
]
});
return;
}
const labels = data.map(x => x.bucket);
const values = data.map(x => x.value);
const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart;
const options = { scales: { y: { beginAtZero: true } } };
if (rep.chart === 'stackedBar') {
options.scales.x = { stacked: true };
options.scales.y.stacked = true;
}
const dataset = {
label: rep.label,
data: values,
borderWidth: 1,
fill: rep.chart !== 'bar' && rep.chart !== 'stackedBar'
};
if (rep.colors) {
dataset.backgroundColor = rep.colors;
dataset.borderColor = rep.colors;
} else if (rep.color) {
dataset.backgroundColor = rep.color;
dataset.borderColor = rep.color;
} else {
dataset.backgroundColor = 'rgba(54, 162, 235, 0.5)';
dataset.borderColor = 'rgba(54, 162, 235, 1)';
}
new Chart(document.getElementById('chart-' + rep.name), {
type: chartType,
data: {
labels: labels,
datasets: [dataset]
},
options: options
});
});
}
function loadReports() {
let path = currentInterval; let path = currentInterval;
if (currentDomain) { if (currentDomain) {
path = 'domains/' + currentDomain + '/' + currentInterval; path = 'domains/' + currentDomain + '/' + currentInterval;
} }
iframe.src = path + '/index.html'; fetch(path + '/reports.json')
.then(r => r.json())
.then(reports => {
container.innerHTML = '';
reports.forEach(rep => {
fetch(path + '/' + rep.html)
.then(r => r.text())
.then(html => {
container.insertAdjacentHTML('beforeend', html);
initReport(rep, path);
});
});
});
} }
intervalSelect.addEventListener('change', () => { intervalSelect.addEventListener('change', () => {
currentInterval = intervalSelect.value; currentInterval = intervalSelect.value;
updateFrame(); loadReports();
}); });
domainSelect.addEventListener('change', () => { domainSelect.addEventListener('change', () => {
currentDomain = domainSelect.value; currentDomain = domainSelect.value;
updateFrame(); loadReports();
}); });
iframe.addEventListener('load', () => { loadReports();
try {
iframe.style.height = iframe.contentDocument.body.scrollHeight + 'px';
} catch (e) {}
});
updateFrame();
feather.replace(); feather.replace();
</script> </script>
</body> </body>

84
templates/report.html
View file

@ -1,84 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>{{ interval.title() }} Report</title>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
<link rel="stylesheet" href="https://cdn.datatables.net/1.13.4/css/jquery.dataTables.min.css">
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/jquery@3.7.0/dist/jquery.min.js"></script>
<script src="https://cdn.datatables.net/1.13.4/js/jquery.dataTables.min.js"></script>
</head>
<body class="section">
<div class="container">
<h1 class="title">{{ interval.title() }} Report</h1>
{% for report in reports %}
<div class="box">
<h2 class="subtitle">{{ report.label }}</h2>
{% if report.chart == 'table' %}
<table id="table-{{ report.name }}" class="table is-striped"></table>
{% else %}
<canvas id="chart-{{ report.name }}"></canvas>
{% endif %}
</div>
{% endfor %}
</div>
<script>
const reports = {{ reports | tojson }};
reports.forEach(rep => {
fetch(rep.json)
.then(r => r.json())
.then(data => {
if (rep.chart === 'table') {
const rows = data.map(x => [x.bucket, x.value]);
new DataTable('#table-' + rep.name, {
data: rows,
columns: [
{ title: 'Bucket' },
{ title: 'Value' }
]
});
return;
}
const labels = data.map(x => x.bucket);
const values = data.map(x => x.value);
const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart;
const options = {
scales: {
y: { beginAtZero: true }
}
};
if (rep.chart === 'stackedBar') {
options.scales.x = { stacked: true };
options.scales.y.stacked = true;
}
const dataset = {
label: rep.label,
data: values,
borderWidth: 1,
fill: rep.chart !== 'bar' && rep.chart !== 'stackedBar'
};
if (rep.colors) {
dataset.backgroundColor = rep.colors;
dataset.borderColor = rep.colors;
} else if (rep.color) {
dataset.backgroundColor = rep.color;
dataset.borderColor = rep.color;
} else {
dataset.backgroundColor = 'rgba(54, 162, 235, 0.5)';
dataset.borderColor = 'rgba(54, 162, 235, 1)';
}
new Chart(document.getElementById('chart-' + rep.name), {
type: chartType,
data: {
labels: labels,
datasets: [dataset]
},
options: options
});
});
});
</script>
</body>
</html>

8
templates/report_snippet.html Normal file
View file

@ -0,0 +1,8 @@
<div class="box">
<h2 class="subtitle">{{ report.label }}</h2>
{% if report.chart == 'table' %}
<table id="table-{{ report.name }}" class="table is-striped"></table>
{% else %}
<canvas id="chart-{{ report.name }}"></canvas>
{% endif %}
</div>

3
tests/test_reports.py
View file

@ -106,6 +106,9 @@ def test_generate_interval(tmp_path, sample_reports, monkeypatch):
assert error_rate[0]["value"] == pytest.approx(50.0) assert error_rate[0]["value"] == pytest.approx(50.0)
reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text()) reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text())
assert {r["name"] for r in reports} == {"hits", "error_rate"} assert {r["name"] for r in reports} == {"hits", "error_rate"}
for r in reports:
snippet = tmp_path / "output" / "hourly" / r["html"]
assert snippet.exists()
def test_generate_interval_domain_filter(tmp_path, sample_reports, monkeypatch): def test_generate_interval_domain_filter(tmp_path, sample_reports, monkeypatch):