From 91f87689d07039ef73527f22727c3abcf568e408 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 04:57:20 -0500 Subject: [PATCH 01/17] ci: add Forgejo Actions workflow for lint, test, and sample reports artifact --- .forgejo/workflows/ci.yml | 92 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .forgejo/workflows/ci.yml diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..65439eb --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,92 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint-and-test: + name: Lint and test (py${{ matrix.python }}) + # Adjust this label to match your Forgejo runner + runs-on: docker + container: + image: python:${{ matrix.python }}-bookworm + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.11", "3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python (inside container) + run: | + python -m pip install --upgrade pip + pip --version + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest + + - name: Format check (black) + run: black --check . + + - name: Lint (flake8) + run: flake8 . + + - name: Run tests (pytest) + env: + PYTHONDONTWRITEBYTECODE: "1" + run: pytest -q + + build-reports: + name: Build sample reports artifact + needs: lint-and-test + runs-on: docker + container: + image: python:3.11-bookworm + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Seed minimal DB and generate reports + run: | + python - <<'PY' + import sqlite3, pathlib + db = pathlib.Path('database/ngxstat.db') + db.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db) + cur = conn.cursor() + cur.execute('''CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY, + ip TEXT, + host TEXT, + time TEXT, + request TEXT, + status INTEGER, + bytes_sent INTEGER, + referer TEXT, + user_agent TEXT, + cache_status TEXT + )''') + cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:00:00','GET / HTTP/1.1',200,100,'-','curl','MISS')") + cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:05:00','GET /about HTTP/1.1',200,100,'-','curl','MISS')") + conn.commit(); conn.close() + PY + python scripts/generate_reports.py global + python scripts/generate_reports.py hourly + python scripts/generate_reports.py index + + - name: Upload reports artifact + uses: actions/upload-artifact@v4 + with: + name: ngxstat-reports + path: output/ + From 0363c372027d1df49ef9fd20ea642f4e065a989d Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:05:33 -0500 Subject: [PATCH 02/17] ci: replace Node-based actions with manual git clone and Debian container --- .forgejo/workflows/ci.yml | 115 +++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 44 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 65439eb..a5f4930 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -6,58 +6,90 @@ on: workflow_dispatch: jobs: - lint-and-test: - name: Lint and test (py${{ matrix.python }}) - # Adjust this label to match your Forgejo runner + ci: + name: Lint, test, and build + # This label must match your Forgejo runner's label runs-on: docker - container: - image: python:${{ matrix.python }}-bookworm - strategy: - fail-fast: false - matrix: - python: ["3.10", "3.11", "3.12"] + # Use a clean Debian container so tools are predictable + container: debian:stable-slim + env: + PYTHONDONTWRITEBYTECODE: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" + UV_SYSTEM_PYTHON: "1" steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python (inside container) + - name: Install build tooling run: | + set -euo pipefail + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git ca-certificates python3 python3-venv python3-pip python3-setuptools \ + python3-wheel sqlite3 + update-ca-certificates || true + + - name: Checkout repository (manual) + run: | + set -euo pipefail + if [ -f Makefile ] || [ -d .git ]; then + echo "Repository present in workspace; skipping clone" + exit 0 + fi + REMOTE_URL="${CI_REPOSITORY_URL:-}" + if [ -z "$REMOTE_URL" ]; then + if [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then + REMOTE_URL="${GITHUB_SERVER_URL%/}/${GITHUB_REPOSITORY}.git" + elif [ -n "${GITHUB_REPOSITORY:-}" ]; then + REMOTE_URL="https://git.jordanwages.com/${GITHUB_REPOSITORY}.git" + else + echo "Unable to determine repository URL from CI environment" >&2 + exit 1 + fi + fi + AUTH_URL="$REMOTE_URL" + if [ -n "${GITHUB_TOKEN:-}" ]; then + ACTOR="${GITHUB_ACTOR:-oauth2}" + AUTH_URL=$(printf '%s' "$REMOTE_URL" | sed -E "s#^https://#https://${ACTOR}:${GITHUB_TOKEN}@#") + fi + echo "Cloning from: $REMOTE_URL" + if ! git clone --depth 1 "$AUTH_URL" .; then + echo "Auth clone failed; trying anonymous clone..." >&2 + git clone --depth 1 "$REMOTE_URL" . + fi + if [ -n "${GITHUB_SHA:-}" ]; then + git fetch --depth 1 origin "$GITHUB_SHA" || true + git checkout -q "$GITHUB_SHA" || true + elif [ -n "${GITHUB_REF_NAME:-}" ]; then + git fetch --depth 1 origin "$GITHUB_REF_NAME" || true + git checkout -q "$GITHUB_REF_NAME" || true + fi + + - name: Set up venv and install deps + run: | + set -euo pipefail + python3 -m venv .venv + . .venv/bin/activate python -m pip install --upgrade pip - pip --version - - - name: Install dependencies - run: | pip install -r requirements.txt pip install pytest - name: Format check (black) - run: black --check . + run: | + . .venv/bin/activate + black --check . - name: Lint (flake8) - run: flake8 . + run: | + . .venv/bin/activate + flake8 . - name: Run tests (pytest) - env: - PYTHONDONTWRITEBYTECODE: "1" - run: pytest -q - - build-reports: - name: Build sample reports artifact - needs: lint-and-test - runs-on: docker - container: - image: python:3.11-bookworm - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + . .venv/bin/activate + pytest -q - - name: Seed minimal DB and generate reports + - name: Build sample reports (no artifact upload) run: | + set -euo pipefail + . .venv/bin/activate python - <<'PY' import sqlite3, pathlib db = pathlib.Path('database/ngxstat.db') @@ -83,10 +115,5 @@ jobs: python scripts/generate_reports.py global python scripts/generate_reports.py hourly python scripts/generate_reports.py index - - - name: Upload reports artifact - uses: actions/upload-artifact@v4 - with: - name: ngxstat-reports - path: output/ - + tar -czf ngxstat-reports.tar.gz -C output . + echo "Built sample reports archive: ngxstat-reports.tar.gz" From 979fbb0e64348adbf4c0a13ba34172275cc6a1e2 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:14:01 -0500 Subject: [PATCH 03/17] ci(lint): configure flake8 excludes/line-length; fix F541 and F401 in tests --- .flake8 | 6 ++++++ tests/test_nginx_config.py | 2 +- tests/test_run_analysis.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..0ba73e9 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +exclude = .git, .venv, output, static/icons +max-line-length = 160 +per-file-ignores = + tests/test_*.py: E402 + diff --git a/tests/test_nginx_config.py b/tests/test_nginx_config.py index a6494a7..b2b1317 100644 --- a/tests/test_nginx_config.py +++ b/tests/test_nginx_config.py @@ -1,6 +1,6 @@ import sys from pathlib import Path -import pytest +import pytest # noqa: F401 REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.append(str(REPO_ROOT)) diff --git a/tests/test_run_analysis.py b/tests/test_run_analysis.py index fad7953..7150895 100644 --- a/tests/test_run_analysis.py +++ b/tests/test_run_analysis.py @@ -16,7 +16,7 @@ def test_script_invokes_commands(tmp_path): python_stub = tmp_path / "python" python_stub.write_text(f'#!/usr/bin/env bash\necho "$*" >> "{calls}"\n') python_stub.chmod(0o755) - (tmp_path / "python3").write_text(f"#!/usr/bin/env bash\nexit 0\n") + (tmp_path / "python3").write_text("#!/usr/bin/env bash\nexit 0\n") (tmp_path / "python3").chmod(0o755) env = os.environ.copy() From 136b4196ea66b9453d776b5fb2d3c03a7a7e0ae3 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:15:59 -0500 Subject: [PATCH 04/17] ci: add pytest --maxfail=1 and simple pip/venv cache at /cache if available --- .forgejo/workflows/ci.yml | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index a5f4930..2717345 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -65,11 +65,33 @@ jobs: - name: Set up venv and install deps run: | set -euo pipefail - python3 -m venv .venv + # Prefer persistent cache if runner provides /cache + USE_CACHE=0 + if [ -d /cache ] && [ -w /cache ]; then + export PIP_CACHE_DIR=/cache/pip + mkdir -p "$PIP_CACHE_DIR" + REQ_HASH=$(sha256sum requirements.txt | awk '{print $1}') + PYVER=$(python3 -c 'import sys;print(".".join(map(str, sys.version_info[:2])))') + CACHE_VENV="/cache/venv-${REQ_HASH}-py${PYVER}" + if [ -d "$CACHE_VENV" ]; then + echo "Using cached virtualenv: $CACHE_VENV" + ln -s "$CACHE_VENV" .venv + USE_CACHE=1 + else + echo "Creating cached virtualenv: $CACHE_VENV" + python3 -m venv "$CACHE_VENV" + ln -s "$CACHE_VENV" .venv + fi + fi + . .venv/bin/activate python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest + if [ "$USE_CACHE" = "1" ]; then + # Ensure required packages are present; pip will use cache + pip install -r requirements.txt pytest || pip install -r requirements.txt pytest + else + pip install -r requirements.txt pytest + fi - name: Format check (black) run: | @@ -84,7 +106,7 @@ jobs: - name: Run tests (pytest) run: | . .venv/bin/activate - pytest -q + pytest -q --maxfail=1 - name: Build sample reports (no artifact upload) run: | From ab4f017ba815d5b364403a87f4c41a1e866f8302 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:17:51 -0500 Subject: [PATCH 05/17] ci: robust venv creation; verify cached venv has activate and fallback to local --- .forgejo/workflows/ci.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 2717345..cd7fb36 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -73,15 +73,24 @@ jobs: REQ_HASH=$(sha256sum requirements.txt | awk '{print $1}') PYVER=$(python3 -c 'import sys;print(".".join(map(str, sys.version_info[:2])))') CACHE_VENV="/cache/venv-${REQ_HASH}-py${PYVER}" - if [ -d "$CACHE_VENV" ]; then - echo "Using cached virtualenv: $CACHE_VENV" - ln -s "$CACHE_VENV" .venv - USE_CACHE=1 - else - echo "Creating cached virtualenv: $CACHE_VENV" + if [ ! -f "$CACHE_VENV/bin/activate" ]; then + echo "Preparing cached virtualenv: $CACHE_VENV" + rm -rf "$CACHE_VENV" || true python3 -m venv "$CACHE_VENV" - ln -s "$CACHE_VENV" .venv fi + ln -sfn "$CACHE_VENV" .venv + USE_CACHE=1 + else + # Fallback to local venv + python3 -m venv .venv + fi + + # If the link didn't produce an activate file, fallback to local venv + if [ ! -f .venv/bin/activate ]; then + echo "Cached venv missing; creating local .venv" + rm -f .venv + python3 -m venv .venv + USE_CACHE=0 fi . .venv/bin/activate From 176359d0100b8d52370f0a154ea22a59462a4e44 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:19:48 -0500 Subject: [PATCH 06/17] lint: remove unused typing import; mark test pytest import as noqa F401 --- scripts/analyze.py | 2 +- tests/test_analyze.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/analyze.py b/scripts/analyze.py index fe7b818..7c4c141 100644 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -18,7 +18,7 @@ from __future__ import annotations import sqlite3 from pathlib import Path -from typing import Dict, List, Optional, Set +from typing import List, Optional, Set from datetime import datetime, timedelta import json diff --git a/tests/test_analyze.py b/tests/test_analyze.py index a4358d7..138e73d 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -3,7 +3,7 @@ import json import sqlite3 from pathlib import Path -import pytest +import pytest # noqa: F401 REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.append(str(REPO_ROOT)) From 5053a4c4db726b81c614e3a647356c833ac2ff26 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:24:14 -0500 Subject: [PATCH 07/17] lint: re-enable E402; remove sys.path hacks; drop unused pytest imports in tests --- .flake8 | 3 --- tests/test_analyze.py | 6 ------ tests/test_nginx_config.py | 4 ---- tests/test_reports.py | 8 +++----- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/.flake8 b/.flake8 index 0ba73e9..95f9808 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,3 @@ [flake8] exclude = .git, .venv, output, static/icons max-line-length = 160 -per-file-ignores = - tests/test_*.py: E402 - diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 138e73d..6e97ab6 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -1,12 +1,6 @@ -import sys import json import sqlite3 from pathlib import Path - -import pytest # noqa: F401 - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.append(str(REPO_ROOT)) from scripts import analyze from scripts import generate_reports as gr diff --git a/tests/test_nginx_config.py b/tests/test_nginx_config.py index b2b1317..604b700 100644 --- a/tests/test_nginx_config.py +++ b/tests/test_nginx_config.py @@ -1,9 +1,5 @@ -import sys from pathlib import Path -import pytest # noqa: F401 -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.append(str(REPO_ROOT)) from scripts import nginx_config as nc diff --git a/tests/test_reports.py b/tests/test_reports.py index fa8f0a0..f6c6918 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,14 +1,10 @@ import sqlite3 from pathlib import Path import json -import sys from datetime import datetime import pytest from typer.testing import CliRunner - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.append(str(REPO_ROOT)) from scripts import generate_reports as gr @@ -205,7 +201,9 @@ def test_generate_root_index(tmp_path, sample_reports, monkeypatch): def test_generated_marker_written(tmp_path, monkeypatch): out_dir = tmp_path / "output" monkeypatch.setattr(gr, "OUTPUT_DIR", out_dir) - monkeypatch.setattr(gr, "TEMPLATE_DIR", REPO_ROOT / "templates") + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) monkeypatch.setattr(gr, "GENERATED_MARKER", out_dir / "generated.txt") monkeypatch.setattr(gr, "_copy_icons", lambda: None) (out_dir / "hourly").mkdir(parents=True) From a8f7ac9b7a53fe920dbe28421a076dd4ff6fe5bf Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:27:02 -0500 Subject: [PATCH 08/17] lint: remove unused Path import in tests/test_nginx_config.py --- tests/test_nginx_config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_nginx_config.py b/tests/test_nginx_config.py index 604b700..cba4212 100644 --- a/tests/test_nginx_config.py +++ b/tests/test_nginx_config.py @@ -1,5 +1,3 @@ -from pathlib import Path - from scripts import nginx_config as nc From 9c26ae3e9027be613f723acac3c654504b1b702e Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Sat, 16 Aug 2025 05:29:20 -0500 Subject: [PATCH 09/17] ci: ensure repo root on PYTHONPATH when running pytest --- .forgejo/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index cd7fb36..5cf26be 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -115,6 +115,7 @@ jobs: - name: Run tests (pytest) run: | . .venv/bin/activate + export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" pytest -q --maxfail=1 - name: Build sample reports (no artifact upload) From fab91d2e04ef2b0fe08c15f666516a20cc11e04e Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Mon, 18 Aug 2025 23:01:00 -0500 Subject: [PATCH 10/17] Phase 1 UX + JS transforms: tabs, windowing, percent/grouping, smoothing, stacked series, metadata pass-through, top_n - Replace tabs with Recent/Trends/Distribution/Tables/Analysis and add sticky controls (interval, domain, window [default 7d], percent, group small, exclude '-' -> Uncached, smoothing toggle). - Client-side transforms: time-window slicing, percent mode, group others (3%), per-report exclusions; stackedBar multi-series; moving average for error_rate. - Generator: pass through optional UX metadata (windows_supported, window_default, group_others_threshold, exclude_values, top_n, stacked, palette) and enforce top_n LIMIT for table reports. - Reports: add status_classes_timeseries and cache_status_timeseries; apply top_n=50 to heavy tables. - Chart manager: add helpers (sliceWindow, excludeValues, toPercent, groupOthers, movingAverage). - URL state + localStorage for context; per-tab filtering for Trends/Distribution/Tables. --- reports.yml | 41 +++++ scripts/generate_reports.py | 44 +++++ static/chartManager.js | 60 ++++++ templates/index.html | 356 ++++++++++++++++++++++++++++++------ 4 files changed, 442 insertions(+), 59 deletions(-) diff --git a/reports.yml b/reports.yml index 1ae8e6f..709d686 100644 --- a/reports.yml +++ b/reports.yml @@ -48,6 +48,7 @@ label: Top Domains icon: globe chart: table + top_n: 50 per_domain: false bucket: domain bucket_label: Domain @@ -75,6 +76,7 @@ label: Top Paths icon: map chart: table + top_n: 50 buckets: - domain - path @@ -102,6 +104,7 @@ label: User Agents icon: user chart: table + top_n: 50 buckets: - domain - user_agent @@ -127,6 +130,7 @@ label: Referrers icon: link chart: table + top_n: 50 buckets: - domain - referrer @@ -170,3 +174,40 @@ - "#209cee" - "#ffdd57" - "#f14668" + +# New time-series: status classes over time (stacked) +- name: status_classes_timeseries + label: Status Classes Over Time + icon: server + chart: stackedBar + bucket: time_bucket + bucket_label: Time + stacked: true + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS "2xx", + SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END) AS "3xx", + SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS "4xx", + SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS "5xx", + COUNT(*) AS total + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket + +# New time-series: cache status over time (compact Hit/Miss; exclude '-' by default) +- name: cache_status_timeseries + label: Cache Status Over Time + icon: archive + chart: stackedBar + bucket: time_bucket + bucket_label: Time + stacked: true + exclude_values: ["-"] + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) AS hit, + SUM(CASE WHEN cache_status = 'MISS' THEN 1 ELSE 0 END) AS miss, + COUNT(*) AS total + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index a45e4eb..073e0b7 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -178,6 +178,16 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: name = definition["name"] query = definition["query"].replace("{bucket}", bucket) query = query.replace("FROM logs", "FROM logs_view") + # Apply top_n limit for tables (performance-friendly), if configured + top_n = definition.get("top_n") + chart_type = definition.get("chart", "line") + if top_n and chart_type == "table": + try: + n = int(top_n) + if "LIMIT" not in query.upper(): + query = f"{query}\nLIMIT {n}" + except Exception: + pass cur.execute(query) rows = cur.fetchall() headers = [c[0] for c in cur.description] @@ -203,6 +213,18 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None: entry["color"] = definition["color"] if "colors" in definition: entry["colors"] = definition["colors"] + # Optional UX metadata passthrough for frontend-only transforms + for key in ( + "windows_supported", + "window_default", + "group_others_threshold", + "exclude_values", + "top_n", + "stacked", + "palette", + ): + if key in definition: + entry[key] = definition[key] _render_snippet(entry, out_dir) report_list.append(entry) @@ -266,6 +288,16 @@ def _generate_global() -> None: name = definition["name"] query = definition["query"] + # Apply top_n limit for tables (performance-friendly), if configured + top_n = definition.get("top_n") + chart_type = definition.get("chart", "line") + if top_n and chart_type == "table": + try: + n = int(top_n) + if "LIMIT" not in query.upper(): + query = f"{query}\nLIMIT {n}" + except Exception: + pass cur.execute(query) rows = cur.fetchall() headers = [c[0] for c in cur.description] @@ -291,6 +323,18 @@ def _generate_global() -> None: entry["color"] = definition["color"] if "colors" in definition: entry["colors"] = definition["colors"] + # Optional UX metadata passthrough for frontend-only transforms + for key in ( + "windows_supported", + "window_default", + "group_others_threshold", + "exclude_values", + "top_n", + "stacked", + "palette", + ): + if key in definition: + entry[key] = definition[key] _render_snippet(entry, out_dir) report_list.append(entry) diff --git a/static/chartManager.js b/static/chartManager.js index 79d83fc..2f14f4f 100644 --- a/static/chartManager.js +++ b/static/chartManager.js @@ -47,3 +47,63 @@ export function reset(container) { }); container.innerHTML = ''; } + +// ---- Lightweight client-side data helpers ---- + +// Slice last N rows from a time-ordered array +export function sliceWindow(data, n) { + if (!Array.isArray(data) || n === undefined || n === null) return data; + if (n === 'all') return data; + const count = Number(n); + if (!Number.isFinite(count) || count <= 0) return data; + return data.slice(-count); +} + +// Exclude rows whose value in key is in excluded list +export function excludeValues(data, key, excluded = []) { + if (!excluded || excluded.length === 0) return data; + const set = new Set(excluded); + return data.filter(row => !set.has(row[key])); +} + +// Compute percentages for categorical distributions (valueKey default 'value') +export function toPercent(data, valueKey = 'value') { + const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0); + if (total <= 0) return data.map(r => ({ ...r })); + return data.map(r => ({ ...r, [valueKey]: (Number(r[valueKey]) || 0) * 100 / total })); +} + +// Group categories with share < threshold into an 'Other' bucket. +export function groupOthers(data, bucketKey, valueKey = 'value', threshold = 0.03, otherLabel = 'Other') { + if (!Array.isArray(data) || data.length === 0) return data; + const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0); + if (total <= 0) return data; + const major = []; + let other = 0; + for (const r of data) { + const v = Number(r[valueKey]) || 0; + if (total && v / total < threshold) { + other += v; + } else { + major.push({ ...r }); + } + } + if (other > 0) major.push({ [bucketKey]: otherLabel, [valueKey]: other }); + return major; +} + +// Simple moving average over numeric array +export function movingAverage(series, span = 3) { + const n = Math.max(1, Number(span) || 1); + const out = []; + for (let i = 0; i < series.length; i++) { + const start = Math.max(0, i - n + 1); + let sum = 0, cnt = 0; + for (let j = start; j <= i; j++) { + const v = Number(series[j]); + if (Number.isFinite(v)) { sum += v; cnt++; } + } + out.push(cnt ? sum / cnt : null); + } + return out; +} diff --git a/templates/index.html b/templates/index.html index edb53f6..56dfd6f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -12,14 +12,15 @@ -
+
+ + + +
-
+
-

Overview

+

Recent

Total logs: -

Date range: - to -

Unique domains: -

@@ -55,13 +88,17 @@
-
@@ -122,6 +125,7 @@ groupOthers, movingAverage, } from './chartManager.js'; + const STATE_KEY = 'ngxstat-state-v2'; const intervalSelect = document.getElementById('interval-select'); const domainSelect = document.getElementById('domain-select'); const intervalControl = document.getElementById('interval-control'); @@ -131,6 +135,7 @@ const modeGroupControl = document.getElementById('mode-group-control'); const excludeUncachedControl = document.getElementById('exclude-uncached-control'); const smoothControl = document.getElementById('smooth-control'); + const resetButton = document.getElementById('reset-view'); const tabs = document.querySelectorAll('#report-tabs li'); const sections = { recent: document.getElementById('recent-section'), @@ -172,10 +177,11 @@ let modeGroup = true; let excludeUncached = true; let smoothError = false; + let hadExplicitWindow = false; // URL or saved-state provided window function saveState() { try { - localStorage.setItem('ngxstat-state', JSON.stringify({ + localStorage.setItem(STATE_KEY, JSON.stringify({ tab: currentTab, interval: currentInterval, domain: currentDomain, @@ -190,11 +196,11 @@ function loadSavedState() { try { - const s = JSON.parse(localStorage.getItem('ngxstat-state') || '{}'); + const s = JSON.parse(localStorage.getItem(STATE_KEY) || '{}'); if (s.tab) currentTab = s.tab; if (s.interval) currentInterval = s.interval; if (s.domain !== undefined) currentDomain = s.domain; - if (s.window) currentWindow = s.window; + if (s.window) { currentWindow = s.window; hadExplicitWindow = true; } if (s.percent !== undefined) modePercent = !!Number(s.percent); if (s.group !== undefined) modeGroup = !!Number(s.group); if (s.exclude_dash !== undefined) excludeUncached = !!Number(s.exclude_dash); @@ -207,7 +213,7 @@ if (params.get('tab')) currentTab = params.get('tab'); if (params.get('interval')) currentInterval = params.get('interval'); if (params.get('domain') !== null) currentDomain = params.get('domain') || ''; - if (params.get('window')) currentWindow = params.get('window'); + if (params.get('window')) { currentWindow = params.get('window'); hadExplicitWindow = true; } if (params.get('percent') !== null) modePercent = params.get('percent') === '1'; if (params.get('group') !== null) modeGroup = params.get('group') === '1'; if (params.get('exclude_dash') !== null) excludeUncached = params.get('exclude_dash') === '1'; @@ -273,8 +279,13 @@ } // Windowing for time series if (isTimeSeries) { - const n = bucketsForWindow(currentWindow, currentInterval); - transformed = sliceWindow(transformed, n); + // Only apply windowing if report supports current window (if constrained) + const supported = Array.isArray(rep.windows_supported) ? rep.windows_supported : null; + const canWindow = !supported || supported.includes(currentWindow); + if (canWindow) { + const n = bucketsForWindow(currentWindow, currentInterval); + transformed = sliceWindow(transformed, n); + } } // Distributions: percent + group small const isDistribution = ['pie', 'polarArea', 'doughnut', 'donut'].includes(rep.chart); @@ -306,7 +317,7 @@ options.scales.y.stacked = true; // Build multiple series from columns (exclude bucket & total) const keys = transformed.length ? Object.keys(transformed[0]).filter(k => k !== bucketField && k !== 'total') : []; - const palette = rep.colors || [ + const palette = rep.colors || rep.palette || [ '#3273dc', '#23d160', '#ffdd57', '#ff3860', '#7957d5', '#363636' ]; datasets = keys.map((k, i) => ({ @@ -327,6 +338,9 @@ if (rep.colors) { dataset.backgroundColor = rep.colors; dataset.borderColor = rep.colors; + } else if (rep.palette) { + dataset.backgroundColor = rep.palette; + dataset.borderColor = rep.palette; } else if (rep.color) { dataset.backgroundColor = rep.color; dataset.borderColor = rep.color; @@ -392,6 +406,15 @@ if (currentTab === 'tables') return rep.chart === 'table'; return true; }); + // If no explicit window was given (URL or saved state), honor first report's default + if (!hadExplicitWindow) { + const withDefault = filtered.find(r => r.window_default); + if (withDefault && typeof withDefault.window_default === 'string') { + currentWindow = withDefault.window_default; + windowSelect.value = currentWindow; + updateURL(); + } + } filtered.forEach(rep => { fetch(path + '/' + rep.html, { signal: token.controller.signal }) .then(r => r.text()) @@ -499,10 +522,12 @@ intervalControl.classList.toggle('is-hidden', !showInterval); domainControl.classList.toggle('is-hidden', !showDomain); windowControl.classList.toggle('is-hidden', !showInterval); - modePercentControl.classList.toggle('is-hidden', !showInterval); - modeGroupControl.classList.toggle('is-hidden', !showInterval); - excludeUncachedControl.classList.toggle('is-hidden', !showInterval); - smoothControl.classList.toggle('is-hidden', !showInterval); + // Only show percent/group/exclude toggles on Distribution tab, + // and smoothing only on Trends tab + modePercentControl.classList.toggle('is-hidden', name !== 'distribution'); + modeGroupControl.classList.toggle('is-hidden', name !== 'distribution'); + excludeUncachedControl.classList.toggle('is-hidden', name !== 'distribution'); + smoothControl.classList.toggle('is-hidden', name !== 'trends'); updateURL(); if (name === 'recent') { loadStats(); @@ -570,6 +595,23 @@ switchTab(tab.dataset.tab); }); }); + resetButton.addEventListener('click', () => { + try { + localStorage.removeItem('ngxstat-state'); // clear legacy + localStorage.removeItem(STATE_KEY); + } catch {} + // Reset to hard defaults + currentTab = 'recent'; + currentInterval = intervalSelect.value = intervalSelect.options[0]?.value || currentInterval; + currentDomain = domainSelect.value = ''; + currentWindow = windowSelect.value = '7d'; + modePercent = percentToggle.checked = false; + modeGroup = groupToggle.checked = true; + excludeUncached = excludeUncachedToggle.checked = true; + smoothError = smoothToggle.checked = false; + hadExplicitWindow = false; + switchTab(currentTab); + }); // Initialize state (URL -> localStorage -> defaults) loadSavedState(); applyURLParams(); diff --git a/tests/test_reports.py b/tests/test_reports.py index f6c6918..60a6df6 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -323,3 +323,93 @@ def test_multi_bucket_table(tmp_path, monkeypatch): entry = next(r for r in reports if r["name"] == "multi") assert entry["buckets"] == ["domain", "agent"] assert entry["bucket_label"] == ["Domain", "Agent"] + + +def test_top_n_limit_applied(tmp_path, monkeypatch): + # Prepare DB with many distinct agents + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + conn = sqlite3.connect(db_path) + cur = conn.cursor() + for i in range(10): + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "127.0.0.1", + "example.com", + f"2024-01-01 11:{i:02d}:00", + "GET /x HTTP/1.1", + 200, + 100, + "-", + f"ua-{i}", + "MISS", + ), + ) + conn.commit() + conn.close() + + cfg = tmp_path / "reports.yml" + cfg.write_text( + """ +- name: agents + chart: table + global: true + top_n: 3 + query: | + SELECT user_agent AS agent, COUNT(*) AS value + FROM logs + GROUP BY user_agent + ORDER BY value DESC +""" + ) + + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output") + monkeypatch.setattr(gr, "REPORT_CONFIG", cfg) + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) + + gr._generate_global() + + data = json.loads((tmp_path / "output" / "global" / "agents.json").read_text()) + # Should be limited to 3 rows + assert len(data) <= 3 + + +def test_metadata_passthrough(tmp_path, monkeypatch): + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + + cfg = tmp_path / "reports.yml" + cfg.write_text( + """ +- name: custom_ts + label: Custom TS + chart: line + window_default: 24h + windows_supported: [1h, 24h, 7d] + palette: ["#111111", "#222222"] + query: | + SELECT {bucket} AS time_bucket, COUNT(*) AS value + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket +""" + ) + + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output") + monkeypatch.setattr(gr, "REPORT_CONFIG", cfg) + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) + + gr._generate_interval("hourly") + + reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text()) + entry = next(r for r in reports if r["name"] == "custom_ts") + assert entry["window_default"] == "24h" + assert entry["windows_supported"] == ["1h", "24h", "7d"] + assert entry["palette"] == ["#111111", "#222222"] From 95e54359d7cc0289655161d0f271b533f9dcf7a1 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Tue, 19 Aug 2025 00:09:49 -0500 Subject: [PATCH 12/17] UX: unify time selection and simplify controls\n\n- Replace separate Interval + Window with a single Time preset (Last hour/24h/7d/30d/12w/12m/All time)\n- Map presets to sensible grouping (hourly/daily/weekly/monthly) based on available intervals\n- Keep backward compatibility: preserve existing URL/state params; keep legacy controls hidden\n- Add client support for new windows (12w, 12m) in time-bucket slicing\n- Show only relevant controls per tab (Trends: smoothing; Distribution: percent/group/exclude)\n- Streamline reset flow to a sane default (Last 7 days) --- templates/index.html | 133 +++++++++++++++++++++++++++++++++---------- 1 file changed, 102 insertions(+), 31 deletions(-) diff --git a/templates/index.html b/templates/index.html index 5b877ef..7e16527 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,6 +21,7 @@
+ @@ -64,17 +63,17 @@
@@ -99,12 +98,17 @@
- - - @@ -157,6 +159,7 @@ trends: document.getElementById('reports-trends'), breakdown: document.getElementById('reports-breakdown') }; + const recentRow = document.getElementById('recent-row'); const analysisElems = { missing: document.getElementById('analysis-missing'), cache: document.getElementById('analysis-cache'), @@ -453,6 +456,11 @@ path = currentDomain ? ('domains/' + encodeURIComponent(currentDomain) + '/' + currentInterval) : currentInterval; } + // Clear the top row on each load of Recent + if (currentTab === 'recent' && recentRow) { + recentRow.innerHTML = ''; + } + const token = newLoad(container); fetch(path + '/reports.json', { signal: token.controller.signal }) @@ -480,7 +488,15 @@ .then(r => r.text()) .then(html => { if (token !== currentLoad) return; - container.insertAdjacentHTML('beforeend', html); + // On Recent tab, render Cache Status and HTTP Statuses side-by-side + const inTopRow = currentTab === 'recent' && + (rep.name === 'cache_status_breakdown' || rep.name === 'status_distribution'); + if (inTopRow && recentRow) { + const wrapped = `
${html}
`; + recentRow.insertAdjacentHTML('beforeend', wrapped); + } else { + container.insertAdjacentHTML('beforeend', html); + } initReport(token, rep, path); }); }); From 371719799192a775c91c5be1c71a40a50dcbb243 Mon Sep 17 00:00:00 2001 From: ngxstat-bot Date: Tue, 19 Aug 2025 00:51:10 -0500 Subject: [PATCH 17/17] analysis: make suggest_cache and detect_threats pure-callable, add CLI wrappers\n\n- Replace Typer Option defaults with plain Python defaults in functions used by generator/tests\n- Add CLI wrapper commands (, ) that delegate to the pure functions\n- Cast params to int for SQL/timedelta to avoid type issues\n- Resolves OptionInfo errors during run-reports analysis phase --- scripts/analyze.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/analyze.py b/scripts/analyze.py index 7c4c141..9f49978 100644 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -155,10 +155,9 @@ def check_missing_domains( typer.echo(d) -@app.command("suggest-cache") def suggest_cache( - threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"), - json_output: bool = typer.Option(False, "--json", help="Output results as JSON"), + threshold: int = 10, + json_output: bool = False, ) -> None: """Suggest domain/path pairs that could benefit from caching. @@ -191,7 +190,7 @@ def suggest_cache( HAVING miss_count >= ? ORDER BY miss_count DESC """, - (threshold,), + (int(threshold),), ) rows = [r for r in cur.fetchall() if r[0] in no_cache] @@ -211,11 +210,18 @@ def suggest_cache( for item in result: typer.echo(f"{item['host']} {item['path']} {item['misses']}") +@app.command("suggest-cache") +def suggest_cache_cli( + threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"), + json_output: bool = typer.Option(False, "--json", help="Output results as JSON"), +) -> None: + """CLI wrapper for suggest_cache.""" + suggest_cache(threshold=threshold, json_output=json_output) + -@app.command("detect-threats") def detect_threats( - hours: int = typer.Option(1, help="Number of recent hours to analyze"), - ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"), + hours: int = 1, + ip_threshold: int = 100, ) -> None: """Detect potential security threats from recent logs.""" @@ -231,8 +237,8 @@ def detect_threats( max_dt = datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") recent_end = max_dt - recent_start = recent_end - timedelta(hours=hours) - prev_start = recent_start - timedelta(hours=hours) + recent_start = recent_end - timedelta(hours=int(hours)) + prev_start = recent_start - timedelta(hours=int(hours)) prev_end = recent_start fmt = "%Y-%m-%d %H:%M:%S" @@ -339,6 +345,14 @@ def detect_threats( out_path.write_text(json.dumps(report, indent=2)) typer.echo(json.dumps(report)) +@app.command("detect-threats") +def detect_threats_cli( + hours: int = typer.Option(1, help="Number of recent hours to analyze"), + ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"), +) -> None: + """CLI wrapper for detect_threats.""" + detect_threats(hours=hours, ip_threshold=ip_threshold) + if __name__ == "__main__": app()