diff --git a/reports.yml b/reports.yml index 1ae8e6f..709d686 100644 --- a/reports.yml +++ b/reports.yml @@ -48,6 +48,7 @@ label: Top Domains icon: globe chart: table + top_n: 50 per_domain: false bucket: domain bucket_label: Domain @@ -75,6 +76,7 @@ label: Top Paths icon: map chart: table + top_n: 50 buckets: - domain - path @@ -102,6 +104,7 @@ label: User Agents icon: user chart: table + top_n: 50 buckets: - domain - user_agent @@ -127,6 +130,7 @@ label: Referrers icon: link chart: table + top_n: 50 buckets: - domain - referrer @@ -170,3 +174,40 @@ - "#209cee" - "#ffdd57" - "#f14668" + +# New time-series: status classes over time (stacked) +- name: status_classes_timeseries + label: Status Classes Over Time + icon: server + chart: stackedBar + bucket: time_bucket + bucket_label: Time + stacked: true + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS "2xx", + SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END) AS "3xx", + SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS "4xx", + SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS "5xx", + COUNT(*) AS total + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket + +# New time-series: cache status over time (compact Hit/Miss; exclude '-' by default) +- name: cache_status_timeseries + label: Cache Status Over Time + icon: archive + chart: stackedBar + bucket: time_bucket + bucket_label: Time + stacked: true + exclude_values: ["-"] + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) AS hit, + SUM(CASE WHEN cache_status = 'MISS' THEN 1 ELSE 0 END) AS miss, + COUNT(*) AS total + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index a45e4eb..073e0b7 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -178,6 +178,16 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: name = definition["name"] query = definition["query"].replace("{bucket}", bucket) query = query.replace("FROM logs", "FROM logs_view") + # Apply top_n limit for tables (performance-friendly), if configured + top_n = definition.get("top_n") + chart_type = definition.get("chart", "line") + if top_n and chart_type == "table": + try: + n = int(top_n) + if "LIMIT" not in query.upper(): + query = f"{query}\nLIMIT {n}" + except Exception: + pass cur.execute(query) rows = cur.fetchall() headers = [c[0] for c in cur.description] @@ -203,6 +213,18 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: entry["color"] = definition["color"] if "colors" in definition: entry["colors"] = definition["colors"] + # Optional UX metadata passthrough for frontend-only transforms + for key in ( + "windows_supported", + "window_default", + "group_others_threshold", + "exclude_values", + "top_n", + "stacked", + "palette", + ): + if key in definition: + entry[key] = definition[key] _render_snippet(entry, out_dir) report_list.append(entry) @@ -266,6 +288,16 @@ def _generate_global() -> None: name = definition["name"] query = definition["query"] + # Apply top_n limit for tables (performance-friendly), if configured + top_n = definition.get("top_n") + chart_type = definition.get("chart", "line") + if top_n and chart_type == "table": + try: + n = int(top_n) + if "LIMIT" not in query.upper(): + query = f"{query}\nLIMIT {n}" + except Exception: + pass cur.execute(query) rows = cur.fetchall() headers = [c[0] for c in cur.description] @@ -291,6 +323,18 @@ def _generate_global() -> None: entry["color"] = definition["color"] if "colors" in definition: entry["colors"] = definition["colors"] + # Optional UX metadata passthrough for frontend-only transforms + for key in ( + "windows_supported", + "window_default", + "group_others_threshold", + "exclude_values", + "top_n", + "stacked", + "palette", + ): + if key in definition: + entry[key] = definition[key] _render_snippet(entry, out_dir) report_list.append(entry) diff --git a/static/chartManager.js b/static/chartManager.js index 79d83fc..2f14f4f 100644 --- a/static/chartManager.js +++ b/static/chartManager.js @@ -47,3 +47,63 @@ export function reset(container) { }); container.innerHTML = ''; } + +// ---- Lightweight client-side data helpers ---- + +// Slice last N rows from a time-ordered array +export function sliceWindow(data, n) { + if (!Array.isArray(data) || n === undefined || n === null) return data; + if (n === 'all') return data; + const count = Number(n); + if (!Number.isFinite(count) || count <= 0) return data; + return data.slice(-count); +} + +// Exclude rows whose value in key is in excluded list +export function excludeValues(data, key, excluded = []) { + if (!excluded || excluded.length === 0) return data; + const set = new Set(excluded); + return data.filter(row => !set.has(row[key])); +} + +// Compute percentages for categorical distributions (valueKey default 'value') +export function toPercent(data, valueKey = 'value') { + const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0); + if (total <= 0) return data.map(r => ({ ...r })); + return data.map(r => ({ ...r, [valueKey]: (Number(r[valueKey]) || 0) * 100 / total })); +} + +// Group categories with share < threshold into an 'Other' bucket. +export function groupOthers(data, bucketKey, valueKey = 'value', threshold = 0.03, otherLabel = 'Other') { + if (!Array.isArray(data) || data.length === 0) return data; + const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0); + if (total <= 0) return data; + const major = []; + let other = 0; + for (const r of data) { + const v = Number(r[valueKey]) || 0; + if (total && v / total < threshold) { + other += v; + } else { + major.push({ ...r }); + } + } + if (other > 0) major.push({ [bucketKey]: otherLabel, [valueKey]: other }); + return major; +} + +// Simple moving average over numeric array +export function movingAverage(series, span = 3) { + const n = Math.max(1, Number(span) || 1); + const out = []; + for (let i = 0; i < series.length; i++) { + const start = Math.max(0, i - n + 1); + let sum = 0, cnt = 0; + for (let j = start; j <= i; j++) { + const v = Number(series[j]); + if (Number.isFinite(v)) { sum += v; cnt++; } + } + out.push(cnt ? sum / cnt : null); + } + return out; +} diff --git a/templates/index.html b/templates/index.html index edb53f6..56dfd6f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -12,14 +12,15 @@
-
+
+ + + +
-
+
-

Overview

+

Recent

Total logs: -

Date range: - to -

Unique domains: -

@@ -55,13 +88,17 @@
-