diff --git a/reports.yml b/reports.yml index c9b9cd3..1ae8e6f 100644 --- a/reports.yml +++ b/reports.yml @@ -75,47 +75,78 @@ label: Top Paths icon: map chart: table - bucket: path - bucket_label: Path + buckets: + - domain + - path + bucket_label: + - Domain + - Path query: | - SELECT path AS path, - COUNT(*) AS value - FROM ( - SELECT substr(substr(request, instr(request, ' ') + 1), 1, + WITH paths AS ( + SELECT host AS domain, + substr(substr(request, instr(request, ' ') + 1), 1, instr(substr(request, instr(request, ' ') + 1), ' ') - 1) AS path FROM logs + ), ranked AS ( + SELECT domain, path, COUNT(*) AS value, + ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn + FROM paths + GROUP BY domain, path ) - GROUP BY path - ORDER BY value DESC - LIMIT 20 + SELECT domain, path, value + FROM ranked + WHERE rn <= 20 + ORDER BY domain, value DESC - name: user_agents label: User Agents icon: user chart: table - bucket: user_agent - bucket_label: User Agent + buckets: + - domain + - user_agent + bucket_label: + - Domain + - User Agent query: | - SELECT user_agent AS user_agent, - COUNT(*) AS value - FROM logs - GROUP BY user_agent - ORDER BY value DESC - LIMIT 20 + WITH ua AS ( + SELECT host AS domain, user_agent + FROM logs + ), ranked AS ( + SELECT domain, user_agent, COUNT(*) AS value, + ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn + FROM ua + GROUP BY domain, user_agent + ) + SELECT domain, user_agent, value + FROM ranked + WHERE rn <= 20 + ORDER BY domain, value DESC - name: referrers label: Referrers icon: link chart: table - bucket: referrer - bucket_label: Referrer + buckets: + - domain + - referrer + bucket_label: + - Domain + - Referrer query: | - SELECT referer AS referrer, - COUNT(*) AS value - FROM logs - GROUP BY referrer - ORDER BY value DESC - LIMIT 20 + WITH ref AS ( + SELECT host AS domain, referer AS referrer + FROM logs + ), ranked AS ( + SELECT domain, referrer, COUNT(*) AS value, + ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn + FROM ref + GROUP BY domain, referrer + ) + SELECT domain, referrer, value + FROM ranked + WHERE rn <= 20 + ORDER BY domain, value DESC - name: status_distribution label: HTTP Statuses diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index e587e6e..265da2d 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -182,6 +182,8 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: entry["icon"] = definition["icon"] if "bucket" in definition: entry["bucket"] = definition["bucket"] + if "buckets" in definition: + entry["buckets"] = definition["buckets"] if "bucket_label" in definition: entry["bucket_label"] = definition["bucket_label"] if "color" in definition: @@ -268,6 +270,8 @@ def _generate_global() -> None: entry["icon"] = definition["icon"] if "bucket" in definition: entry["bucket"] = definition["bucket"] + if "buckets" in definition: + entry["buckets"] = definition["buckets"] if "bucket_label" in definition: entry["bucket_label"] = definition["bucket_label"] if "color" in definition: diff --git a/templates/index.html b/templates/index.html index 1b27003..edb53f6 100644 --- a/templates/index.html +++ b/templates/index.html @@ -117,21 +117,24 @@ .then(r => r.json()) .then(data => { if (token !== currentLoad) return; - const bucketField = rep.bucket || 'bucket'; + const bucketFields = rep.buckets || [rep.bucket || 'bucket']; + const labels = Array.isArray(rep.bucket_label) + ? rep.bucket_label + : [rep.bucket_label || 'Bucket']; if (rep.chart === 'table') { - const rows = data.map(x => [x[bucketField], x.value]); + const rows = data.map(x => bucketFields.map(f => x[f]).concat(x.value)); + const columns = labels.map(l => ({ title: l })); + columns.push({ title: 'Value' }); const table = new DataTable('#table-' + rep.name, { data: rows, - columns: [ - { title: rep.bucket_label || 'Bucket' }, - { title: 'Value' } - ] + columns: columns }); registerChart(token, rep.name, table); return; } - const labels = data.map(x => x[bucketField]); + const bucketField = bucketFields[0]; + const labelsArr = data.map(x => x[bucketField]); const values = data.map(x => x.value); const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart; const options = { scales: { y: { beginAtZero: true } } }; @@ -158,7 +161,7 @@ const chart = new Chart(document.getElementById('chart-' + rep.name), { type: chartType, data: { - labels: labels, + labels: labelsArr, datasets: [dataset] }, options: options diff --git a/tests/test_reports.py b/tests/test_reports.py index 75d7737..f9399df 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -253,3 +253,59 @@ def test_global_stats_file(tmp_path, sample_reports, monkeypatch): assert stats["unique_domains"] == 1 assert isinstance(stats["generated_at"], str) assert stats["generation_seconds"] >= 0 + + +def test_multi_bucket_table(tmp_path, monkeypatch): + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + # add a second domain entry + conn = sqlite3.connect(db_path) + cur = conn.cursor() + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "127.0.0.1", + "foo.com", + "2024-01-01 10:10:00", + "GET /foo HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ) + conn.commit() + conn.close() + + cfg = tmp_path / "reports.yml" + cfg.write_text( + """ +- name: multi + chart: table + global: true + buckets: [domain, agent] + bucket_label: [Domain, Agent] + query: | + SELECT host AS domain, user_agent AS agent, COUNT(*) AS value + FROM logs + GROUP BY host, agent +""" + ) + + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output") + monkeypatch.setattr(gr, "REPORT_CONFIG", cfg) + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) + + gr._generate_global() + gr._generate_interval("hourly") + + data = json.loads((tmp_path / "output" / "global" / "multi.json").read_text()) + assert {"domain", "agent", "value"} <= data[0].keys() + reports = json.loads((tmp_path / "output" / "global" / "reports.json").read_text()) + entry = next(r for r in reports if r["name"] == "multi") + assert entry["buckets"] == ["domain", "agent"] + assert entry["bucket_label"] == ["Domain", "Agent"]