wagesj45/ngxstat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add multi-bucket support for tables and update reports

Browse source
This commit is contained in:
Jordan Wages 2025-07-19 18:19:58 -05:00
commit 1d4e99c69b
4 changed files with 127 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

81
reports.yml
View file

@ -75,47 +75,78 @@
label: Top Paths
icon: map
chart: table
bucket: path
bucket_label: Path
buckets:
- domain
- path
bucket_label:
- Domain
- Path
query: |
SELECT path AS path,
COUNT(*) AS value
FROM (
SELECT substr(substr(request, instr(request, ' ') + 1), 1,
WITH paths AS (
SELECT host AS domain,
substr(substr(request, instr(request, ' ') + 1), 1,
instr(substr(request, instr(request, ' ') + 1), ' ') - 1) AS path
FROM logs
), ranked AS (
SELECT domain, path, COUNT(*) AS value,
ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn
FROM paths
GROUP BY domain, path
)
GROUP BY path
ORDER BY value DESC
LIMIT 20
SELECT domain, path, value
FROM ranked
WHERE rn <= 20
ORDER BY domain, value DESC
- name: user_agents
label: User Agents
icon: user
chart: table
bucket: user_agent
bucket_label: User Agent
buckets:
- domain
- user_agent
bucket_label:
- Domain
- User Agent
query: |
SELECT user_agent AS user_agent,
COUNT(*) AS value
FROM logs
GROUP BY user_agent
ORDER BY value DESC
LIMIT 20
WITH ua AS (
SELECT host AS domain, user_agent
FROM logs
), ranked AS (
SELECT domain, user_agent, COUNT(*) AS value,
ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn
FROM ua
GROUP BY domain, user_agent
)
SELECT domain, user_agent, value
FROM ranked
WHERE rn <= 20
ORDER BY domain, value DESC
- name: referrers
label: Referrers
icon: link
chart: table
bucket: referrer
bucket_label: Referrer
buckets:
- domain
- referrer
bucket_label:
- Domain
- Referrer
query: |
SELECT referer AS referrer,
COUNT(*) AS value
FROM logs
GROUP BY referrer
ORDER BY value DESC
LIMIT 20
WITH ref AS (
SELECT host AS domain, referer AS referrer
FROM logs
), ranked AS (
SELECT domain, referrer, COUNT(*) AS value,
ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn
FROM ref
GROUP BY domain, referrer
)
SELECT domain, referrer, value
FROM ranked
WHERE rn <= 20
ORDER BY domain, value DESC
- name: status_distribution
label: HTTP Statuses

4
scripts/generate_reports.py
View file

@ -182,6 +182,8 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
entry["icon"] = definition["icon"]
if "bucket" in definition:
entry["bucket"] = definition["bucket"]
if "buckets" in definition:
entry["buckets"] = definition["buckets"]
if "bucket_label" in definition:
entry["bucket_label"] = definition["bucket_label"]
if "color" in definition:
@ -268,6 +270,8 @@ def _generate_global() -> None:
entry["icon"] = definition["icon"]
if "bucket" in definition:
entry["bucket"] = definition["bucket"]
if "buckets" in definition:
entry["buckets"] = definition["buckets"]
if "bucket_label" in definition:
entry["bucket_label"] = definition["bucket_label"]
if "color" in definition:

19
templates/index.html
View file

@ -117,21 +117,24 @@
.then(r => r.json())
.then(data => {
if (token !== currentLoad) return;
const bucketField = rep.bucket || 'bucket';
const bucketFields = rep.buckets || [rep.bucket || 'bucket'];
const labels = Array.isArray(rep.bucket_label)
? rep.bucket_label
: [rep.bucket_label || 'Bucket'];
if (rep.chart === 'table') {
const rows = data.map(x => [x[bucketField], x.value]);
const rows = data.map(x => bucketFields.map(f => x[f]).concat(x.value));
const columns = labels.map(l => ({ title: l }));
columns.push({ title: 'Value' });
const table = new DataTable('#table-' + rep.name, {
data: rows,
columns: [
{ title: rep.bucket_label || 'Bucket' },
{ title: 'Value' }
]
columns: columns
});
registerChart(token, rep.name, table);
return;
}
const labels = data.map(x => x[bucketField]);
const bucketField = bucketFields[0];
const labelsArr = data.map(x => x[bucketField]);
const values = data.map(x => x.value);
const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart;
const options = { scales: { y: { beginAtZero: true } } };
@ -158,7 +161,7 @@
const chart = new Chart(document.getElementById('chart-' + rep.name), {
type: chartType,
data: {
labels: labels,
labels: labelsArr,
datasets: [dataset]
},
options: options

56
tests/test_reports.py
View file

@ -253,3 +253,59 @@ def test_global_stats_file(tmp_path, sample_reports, monkeypatch):
assert stats["unique_domains"] == 1
assert isinstance(stats["generated_at"], str)
assert stats["generation_seconds"] >= 0
def test_multi_bucket_table(tmp_path, monkeypatch):
db_path = tmp_path / "database" / "ngxstat.db"
setup_db(db_path)
# add a second domain entry
conn = sqlite3.connect(db_path)
cur = conn.cursor()
cur.execute(
"INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
"127.0.0.1",
"foo.com",
"2024-01-01 10:10:00",
"GET /foo HTTP/1.1",
200,
100,
"-",
"curl",
"MISS",
),
)
conn.commit()
conn.close()
cfg = tmp_path / "reports.yml"
cfg.write_text(
"""
- name: multi
chart: table
global: true
buckets: [domain, agent]
bucket_label: [Domain, Agent]
query: |
SELECT host AS domain, user_agent AS agent, COUNT(*) AS value
FROM logs
GROUP BY host, agent
"""
)
monkeypatch.setattr(gr, "DB_PATH", db_path)
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
monkeypatch.setattr(
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
)
gr._generate_global()
gr._generate_interval("hourly")
data = json.loads((tmp_path / "output" / "global" / "multi.json").read_text())
assert {"domain", "agent", "value"} <= data[0].keys()
reports = json.loads((tmp_path / "output" / "global" / "reports.json").read_text())
entry = next(r for r in reports if r["name"] == "multi")
assert entry["buckets"] == ["domain", "agent"]
assert entry["bucket_label"] == ["Domain", "Agent"]