Compare commits

..

No commits in common. "main" and "codex/add-domain-grouping-to-reports" have entirely different histories.

13 changed files with 112 additions and 951 deletions

View file

@ -1,3 +0,0 @@
[flake8]
exclude = .git, .venv, output, static/icons
max-line-length = 160

View file

@ -1,151 +0,0 @@
name: CI
on:
push:
pull_request:
workflow_dispatch:
jobs:
ci:
name: Lint, test, and build
# This label must match your Forgejo runner's label
runs-on: docker
# Use a clean Debian container so tools are predictable
container: debian:stable-slim
env:
PYTHONDONTWRITEBYTECODE: "1"
PIP_DISABLE_PIP_VERSION_CHECK: "1"
UV_SYSTEM_PYTHON: "1"
steps:
- name: Install build tooling
run: |
set -euo pipefail
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
git ca-certificates python3 python3-venv python3-pip python3-setuptools \
python3-wheel sqlite3
update-ca-certificates || true
- name: Checkout repository (manual)
run: |
set -euo pipefail
if [ -f Makefile ] || [ -d .git ]; then
echo "Repository present in workspace; skipping clone"
exit 0
fi
REMOTE_URL="${CI_REPOSITORY_URL:-}"
if [ -z "$REMOTE_URL" ]; then
if [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
REMOTE_URL="${GITHUB_SERVER_URL%/}/${GITHUB_REPOSITORY}.git"
elif [ -n "${GITHUB_REPOSITORY:-}" ]; then
REMOTE_URL="https://git.jordanwages.com/${GITHUB_REPOSITORY}.git"
else
echo "Unable to determine repository URL from CI environment" >&2
exit 1
fi
fi
AUTH_URL="$REMOTE_URL"
if [ -n "${GITHUB_TOKEN:-}" ]; then
ACTOR="${GITHUB_ACTOR:-oauth2}"
AUTH_URL=$(printf '%s' "$REMOTE_URL" | sed -E "s#^https://#https://${ACTOR}:${GITHUB_TOKEN}@#")
fi
echo "Cloning from: $REMOTE_URL"
if ! git clone --depth 1 "$AUTH_URL" .; then
echo "Auth clone failed; trying anonymous clone..." >&2
git clone --depth 1 "$REMOTE_URL" .
fi
if [ -n "${GITHUB_SHA:-}" ]; then
git fetch --depth 1 origin "$GITHUB_SHA" || true
git checkout -q "$GITHUB_SHA" || true
elif [ -n "${GITHUB_REF_NAME:-}" ]; then
git fetch --depth 1 origin "$GITHUB_REF_NAME" || true
git checkout -q "$GITHUB_REF_NAME" || true
fi
- name: Set up venv and install deps
run: |
set -euo pipefail
# Prefer persistent cache if runner provides /cache
USE_CACHE=0
if [ -d /cache ] && [ -w /cache ]; then
export PIP_CACHE_DIR=/cache/pip
mkdir -p "$PIP_CACHE_DIR"
REQ_HASH=$(sha256sum requirements.txt | awk '{print $1}')
PYVER=$(python3 -c 'import sys;print(".".join(map(str, sys.version_info[:2])))')
CACHE_VENV="/cache/venv-${REQ_HASH}-py${PYVER}"
if [ ! -f "$CACHE_VENV/bin/activate" ]; then
echo "Preparing cached virtualenv: $CACHE_VENV"
rm -rf "$CACHE_VENV" || true
python3 -m venv "$CACHE_VENV"
fi
ln -sfn "$CACHE_VENV" .venv
USE_CACHE=1
else
# Fallback to local venv
python3 -m venv .venv
fi
# If the link didn't produce an activate file, fallback to local venv
if [ ! -f .venv/bin/activate ]; then
echo "Cached venv missing; creating local .venv"
rm -f .venv
python3 -m venv .venv
USE_CACHE=0
fi
. .venv/bin/activate
python -m pip install --upgrade pip
if [ "$USE_CACHE" = "1" ]; then
# Ensure required packages are present; pip will use cache
pip install -r requirements.txt pytest || pip install -r requirements.txt pytest
else
pip install -r requirements.txt pytest
fi
- name: Format check (black)
run: |
. .venv/bin/activate
black --check .
- name: Lint (flake8)
run: |
. .venv/bin/activate
flake8 .
- name: Run tests (pytest)
run: |
. .venv/bin/activate
export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
pytest -q --maxfail=1
- name: Build sample reports (no artifact upload)
run: |
set -euo pipefail
. .venv/bin/activate
python - <<'PY'
import sqlite3, pathlib
db = pathlib.Path('database/ngxstat.db')
db.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(db)
cur = conn.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY,
ip TEXT,
host TEXT,
time TEXT,
request TEXT,
status INTEGER,
bytes_sent INTEGER,
referer TEXT,
user_agent TEXT,
cache_status TEXT
)''')
cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:00:00','GET / HTTP/1.1',200,100,'-','curl','MISS')")
cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:05:00','GET /about HTTP/1.1',200,100,'-','curl','MISS')")
conn.commit(); conn.close()
PY
python scripts/generate_reports.py global
python scripts/generate_reports.py hourly
python scripts/generate_reports.py index
tar -czf ngxstat-reports.tar.gz -C output .
echo "Built sample reports archive: ngxstat-reports.tar.gz"

View file

@ -39,10 +39,9 @@ all intervals in one go:
```
The script calls `scripts/generate_reports.py` internally to create hourly,
daily, weekly and monthly reports, then writes analysis JSON files used by the
"Analysis" tab. Per-domain reports are written under `output/domains/<domain>`
alongside the aggregate data. Open `output/index.html` in a browser to view the
dashboard.
daily, weekly and monthly reports. Per-domain reports are written under
`output/domains/<domain>` alongside the aggregate data. Open
`output/index.html` in a browser to view the dashboard.
If you prefer to run individual commands you can invoke the generator directly:
@ -55,14 +54,8 @@ python scripts/generate_reports.py daily --all-domains
`run-analysis.sh` executes additional utilities that examine the database for
missing domains, caching opportunities and potential threats. The JSON output is
saved under `output/analysis` and appears in the "Analysis" tab. The
`run-reports.sh` script also generates these JSON files as part of the build.
## UX Controls
The dashboard defaults to a 7day window for time series. Your view preferences
persist locally in the browser under the `ngxstat-state-v2` key. Use the
"Reset view" button to clear saved state and restore defaults.
saved under `output/analysis` and appears in the "Analysis" tab of the
dashboard.
```bash
./run-analysis.sh

View file

@ -48,7 +48,6 @@
label: Top Domains
icon: globe
chart: table
top_n: 50
per_domain: false
bucket: domain
bucket_label: Domain
@ -76,7 +75,6 @@
label: Top Paths
icon: map
chart: table
top_n: 50
buckets:
- domain
- path
@ -104,7 +102,6 @@
label: User Agents
icon: user
chart: table
top_n: 50
buckets:
- domain
- user_agent
@ -130,7 +127,6 @@
label: Referrers
icon: link
chart: table
top_n: 50
buckets:
- domain
- referrer
@ -174,40 +170,3 @@
- "#209cee"
- "#ffdd57"
- "#f14668"
# New time-series: status classes over time (stacked)
- name: status_classes_timeseries
label: Status Classes Over Time
icon: server
chart: stackedBar
bucket: time_bucket
bucket_label: Time
stacked: true
query: |
SELECT {bucket} AS time_bucket,
SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS "2xx",
SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END) AS "3xx",
SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS "4xx",
SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS "5xx",
COUNT(*) AS total
FROM logs
GROUP BY time_bucket
ORDER BY time_bucket
# New time-series: cache status over time (compact Hit/Miss; exclude '-' by default)
- name: cache_status_timeseries
label: Cache Status Over Time
icon: archive
chart: stackedBar
bucket: time_bucket
bucket_label: Time
stacked: true
exclude_values: ["-"]
query: |
SELECT {bucket} AS time_bucket,
SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) AS hit,
SUM(CASE WHEN cache_status = 'MISS' THEN 1 ELSE 0 END) AS miss,
COUNT(*) AS total
FROM logs
GROUP BY time_bucket
ORDER BY time_bucket

View file

@ -29,25 +29,21 @@ fi
# Generate reports for all domains combined
echo "[INFO] Generating aggregate reports..."
python -m scripts.generate_reports hourly
python -m scripts.generate_reports daily
python -m scripts.generate_reports weekly
python -m scripts.generate_reports monthly
python -m scripts.generate_reports global
python scripts/generate_reports.py hourly
python scripts/generate_reports.py daily
python scripts/generate_reports.py weekly
python scripts/generate_reports.py monthly
python scripts/generate_reports.py global
# Generate reports for each individual domain
echo "[INFO] Generating per-domain reports..."
python -m scripts.generate_reports hourly --all-domains
python -m scripts.generate_reports daily --all-domains
python -m scripts.generate_reports weekly --all-domains
python -m scripts.generate_reports monthly --all-domains
# Generate analysis JSON
echo "[INFO] Generating analysis files..."
python -m scripts.generate_reports analysis
python scripts/generate_reports.py hourly --all-domains
python scripts/generate_reports.py daily --all-domains
python scripts/generate_reports.py weekly --all-domains
python scripts/generate_reports.py monthly --all-domains
# Generate root index
python -m scripts.generate_reports index
python scripts/generate_reports.py index
# Deactivate to keep cron environment clean
if type deactivate >/dev/null 2>&1; then

View file

@ -18,7 +18,7 @@ from __future__ import annotations
import sqlite3
from pathlib import Path
from typing import List, Optional, Set
from typing import Dict, List, Optional, Set
from datetime import datetime, timedelta
import json
@ -155,9 +155,10 @@ def check_missing_domains(
typer.echo(d)
@app.command("suggest-cache")
def suggest_cache(
threshold: int = 10,
json_output: bool = False,
threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"),
json_output: bool = typer.Option(False, "--json", help="Output results as JSON"),
) -> None:
"""Suggest domain/path pairs that could benefit from caching.
@ -190,7 +191,7 @@ def suggest_cache(
HAVING miss_count >= ?
ORDER BY miss_count DESC
""",
(int(threshold),),
(threshold,),
)
rows = [r for r in cur.fetchall() if r[0] in no_cache]
@ -210,18 +211,11 @@ def suggest_cache(
for item in result:
typer.echo(f"{item['host']} {item['path']} {item['misses']}")
@app.command("suggest-cache")
def suggest_cache_cli(
threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"),
json_output: bool = typer.Option(False, "--json", help="Output results as JSON"),
) -> None:
"""CLI wrapper for suggest_cache."""
suggest_cache(threshold=threshold, json_output=json_output)
@app.command("detect-threats")
def detect_threats(
hours: int = 1,
ip_threshold: int = 100,
hours: int = typer.Option(1, help="Number of recent hours to analyze"),
ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"),
) -> None:
"""Detect potential security threats from recent logs."""
@ -237,8 +231,8 @@ def detect_threats(
max_dt = datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S")
recent_end = max_dt
recent_start = recent_end - timedelta(hours=int(hours))
prev_start = recent_start - timedelta(hours=int(hours))
recent_start = recent_end - timedelta(hours=hours)
prev_start = recent_start - timedelta(hours=hours)
prev_end = recent_start
fmt = "%Y-%m-%d %H:%M:%S"
@ -345,14 +339,6 @@ def detect_threats(
out_path.write_text(json.dumps(report, indent=2))
typer.echo(json.dumps(report))
@app.command("detect-threats")
def detect_threats_cli(
hours: int = typer.Option(1, help="Number of recent hours to analyze"),
ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"),
) -> None:
"""CLI wrapper for detect_threats."""
detect_threats(hours=hours, ip_threshold=ip_threshold)
if __name__ == "__main__":
app()

View file

@ -1,10 +1,9 @@
import json
import sys
import sqlite3
from pathlib import Path
import shutil
from typing import List, Dict, Optional
from datetime import datetime, timezone
from datetime import datetime
import time
import yaml
@ -12,16 +11,10 @@ import yaml
import typer
from jinja2 import Environment, FileSystemLoader
# Ensure project root is importable when running as a script (python scripts/generate_reports.py)
PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
DB_PATH = Path("database/ngxstat.db")
OUTPUT_DIR = Path("output")
TEMPLATE_DIR = Path("templates")
REPORT_CONFIG = Path("reports.yml")
GENERATED_MARKER = OUTPUT_DIR / "generated.txt"
# Mapping of interval names to SQLite strftime formats. These strings are
# substituted into report queries whenever the special ``{bucket}`` token is
@ -37,19 +30,6 @@ INTERVAL_FORMATS = {
app = typer.Typer(help="Generate aggregated log reports")
@app.callback()
def _cli_callback(ctx: typer.Context) -> None:
"""Register post-command hook to note generation time."""
def _write_marker() -> None:
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
# Use timezone-aware UTC to avoid deprecation warnings and ambiguity
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
GENERATED_MARKER.write_text(f"{timestamp}\n")
ctx.call_on_close(_write_marker)
def _get_domains() -> List[str]:
"""Return a sorted list of unique domains from the logs table."""
conn = sqlite3.connect(DB_PATH)
@ -185,16 +165,6 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
name = definition["name"]
query = definition["query"].replace("{bucket}", bucket)
query = query.replace("FROM logs", "FROM logs_view")
# Apply top_n limit for tables (performance-friendly), if configured
top_n = definition.get("top_n")
chart_type = definition.get("chart", "line")
if top_n and chart_type == "table":
try:
n = int(top_n)
if "LIMIT" not in query.upper():
query = f"{query}\nLIMIT {n}"
except Exception:
pass
cur.execute(query)
rows = cur.fetchall()
headers = [c[0] for c in cur.description]
@ -220,18 +190,6 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
entry["color"] = definition["color"]
if "colors" in definition:
entry["colors"] = definition["colors"]
# Optional UX metadata passthrough for frontend-only transforms
for key in (
"windows_supported",
"window_default",
"group_others_threshold",
"exclude_values",
"top_n",
"stacked",
"palette",
):
if key in definition:
entry[key] = definition[key]
_render_snippet(entry, out_dir)
report_list.append(entry)
@ -278,8 +236,7 @@ def _generate_global() -> None:
return
start_time = time.time()
# Use timezone-aware UTC for generated_at (string remains unchanged format)
generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
generated_at = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
_copy_icons()
@ -296,16 +253,6 @@ def _generate_global() -> None:
name = definition["name"]
query = definition["query"]
# Apply top_n limit for tables (performance-friendly), if configured
top_n = definition.get("top_n")
chart_type = definition.get("chart", "line")
if top_n and chart_type == "table":
try:
n = int(top_n)
if "LIMIT" not in query.upper():
query = f"{query}\nLIMIT {n}"
except Exception:
pass
cur.execute(query)
rows = cur.fetchall()
headers = [c[0] for c in cur.description]
@ -331,18 +278,6 @@ def _generate_global() -> None:
entry["color"] = definition["color"]
if "colors" in definition:
entry["colors"] = definition["colors"]
# Optional UX metadata passthrough for frontend-only transforms
for key in (
"windows_supported",
"window_default",
"group_others_threshold",
"exclude_values",
"top_n",
"stacked",
"palette",
):
if key in definition:
entry[key] = definition[key]
_render_snippet(entry, out_dir)
report_list.append(entry)
@ -352,34 +287,6 @@ def _generate_global() -> None:
typer.echo("Generated global reports")
def _generate_analysis() -> None:
"""Generate analysis JSON files consumed by the Analysis tab."""
try:
# Import lazily to avoid circulars and keep dependencies optional
from scripts import analyze
except Exception as exc: # pragma: no cover - defensive
typer.echo(f"Failed to import analysis module: {exc}")
return
# Ensure output root and icons present for parity
_copy_icons()
# These commands write JSON files under output/analysis/
try:
analyze.check_missing_domains(json_output=True)
except Exception as exc: # pragma: no cover - continue best-effort
typer.echo(f"check_missing_domains failed: {exc}")
try:
analyze.suggest_cache(json_output=True)
except Exception as exc: # pragma: no cover
typer.echo(f"suggest_cache failed: {exc}")
try:
analyze.detect_threats()
except Exception as exc: # pragma: no cover
typer.echo(f"detect_threats failed: {exc}")
typer.echo("Generated analysis JSON files")
@app.command()
def hourly(
domain: Optional[str] = typer.Option(
@ -450,12 +357,6 @@ def global_reports() -> None:
_generate_global()
@app.command()
def analysis() -> None:
"""Generate analysis JSON files for the Analysis tab."""
_generate_analysis()
@app.command()
def index() -> None:
"""Generate the root index page linking all reports."""

View file

@ -47,63 +47,3 @@ export function reset(container) {
});
container.innerHTML = '';
}
// ---- Lightweight client-side data helpers ----
// Slice last N rows from a time-ordered array
export function sliceWindow(data, n) {
if (!Array.isArray(data) || n === undefined || n === null) return data;
if (n === 'all') return data;
const count = Number(n);
if (!Number.isFinite(count) || count <= 0) return data;
return data.slice(-count);
}
// Exclude rows whose value in key is in excluded list
export function excludeValues(data, key, excluded = []) {
if (!excluded || excluded.length === 0) return data;
const set = new Set(excluded);
return data.filter(row => !set.has(row[key]));
}
// Compute percentages for categorical distributions (valueKey default 'value')
export function toPercent(data, valueKey = 'value') {
const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0);
if (total <= 0) return data.map(r => ({ ...r }));
return data.map(r => ({ ...r, [valueKey]: (Number(r[valueKey]) || 0) * 100 / total }));
}
// Group categories with share < threshold into an 'Other' bucket.
export function groupOthers(data, bucketKey, valueKey = 'value', threshold = 0.03, otherLabel = 'Other') {
if (!Array.isArray(data) || data.length === 0) return data;
const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0);
if (total <= 0) return data;
const major = [];
let other = 0;
for (const r of data) {
const v = Number(r[valueKey]) || 0;
if (total && v / total < threshold) {
other += v;
} else {
major.push({ ...r });
}
}
if (other > 0) major.push({ [bucketKey]: otherLabel, [valueKey]: other });
return major;
}
// Simple moving average over numeric array
export function movingAverage(series, span = 3) {
const n = Math.max(1, Number(span) || 1);
const out = [];
for (let i = 0; i < series.length; i++) {
const start = Math.max(0, i - n + 1);
let sum = 0, cnt = 0;
for (let j = start; j <= i; j++) {
const v = Number(series[j]);
if (Number.isFinite(v)) { sum += v; cnt++; }
}
out.push(cnt ? sum / cnt : null);
}
return out;
}

View file

@ -12,15 +12,14 @@
<div class="tabs is-toggle" id="report-tabs">
<ul>
<li class="is-active" data-tab="recent"><a>Recent</a></li>
<li data-tab="trends"><a>Trends</a></li>
<li data-tab="breakdown"><a>Breakdown</a></li>
<li class="is-active" data-tab="overview"><a>Overview</a></li>
<li data-tab="all"><a>All Domains</a></li>
<li data-tab="domain"><a>Per Domain</a></li>
<li data-tab="analysis"><a>Analysis</a></li>
</ul>
</div>
<div id="controls" class="field is-grouped is-align-items-center mb-4" style="position: sticky; top: 0; background: white; z-index: 2; padding: 0.5rem 0;">
<!-- Hidden native interval control kept for compatibility and availability probing -->
<div id="controls" class="field is-grouped mb-4">
<div id="interval-control" class="control has-icons-left is-hidden">
<div class="select is-small">
<select id="interval-select">
@ -42,75 +41,26 @@
</div>
<span class="icon is-small is-left"><img src="icons/server.svg" alt="Domain"></span>
</div>
<!-- Unified Time control: selects both range and sensible grouping -->
<div id="time-control" class="control has-icons-left is-hidden">
<div class="select is-small">
<select id="time-select">
<option value="1h">Last hour</option>
<option value="24h">Last 24 hours</option>
<option value="7d" selected>Last 7 days</option>
<option value="30d">Last 30 days</option>
<option value="12w">Last 12 weeks</option>
<option value="12m">Last 12 months</option>
<option value="all">All time</option>
</select>
</div>
<span class="icon is-small is-left"><img src="icons/clock.svg" alt="Time"></span>
</div>
<div id="smooth-control" class="control is-hidden">
<label class="checkbox is-small">
<input type="checkbox" id="smooth-toggle"> Smooth error rate
</label>
</div>
<div id="mode-percent-control" class="control is-hidden">
<label class="checkbox is-small" title="Show values as a percentage of the total, instead of raw counts.">
<input type="checkbox" id="percent-toggle"> Percent mode
</label>
</div>
<div id="mode-group-control" class="control is-hidden">
<label class="checkbox is-small" title="Combine small categories into an 'Other' slice to declutter charts.">
<input type="checkbox" id="group-toggle" checked> Group small into Other
</label>
</div>
<div id="exclude-uncached-control" class="control is-hidden">
<label class="checkbox is-small" title="Hide uncached entries (cache status '-') from cache status distributions.">
<input type="checkbox" id="exclude-uncached-toggle" checked> Exclude “-”
</label>
</div>
<div id="reset-control" class="control">
<button id="reset-view" class="button is-small is-light">Reset view</button>
</div>
</div>
<div id="recent-section">
<div id="overview-section">
<div id="overview" class="box mb-5">
<h2 class="subtitle">Recent</h2>
<h2 class="subtitle">Overview</h2>
<p>Total logs: <span id="stat-total">-</span></p>
<p>Date range: <span id="stat-start">-</span> to <span id="stat-end">-</span></p>
<p>Unique domains: <span id="stat-domains">-</span></p>
<p>Last generated: <span id="stat-generated">-</span></p>
<p>Generation time: <span id="stat-elapsed">-</span> seconds</p>
</div>
<!-- Two key distributions side-by-side on Recent -->
<div id="recent-row" class="columns"></div>
<div id="overview-reports"></div>
</div>
<div id="trends-section" class="is-hidden">
<div id="reports-trends"></div>
<div id="all-section" class="is-hidden">
<div id="reports-all"></div>
</div>
<div id="breakdown-section" class="is-hidden">
<div class="box mb-4">
<h2 class="subtitle">Breakdown</h2>
<p class="mb-2">Explore categorical distributions and detailed lists side-by-side. Use the options below to adjust how categories are shown.</p>
<ul style="margin-left: 1.2rem; list-style: disc;">
<li><strong>Percent mode</strong>: converts counts into percentages of the total for easier comparison.</li>
<li><strong>Group small into Other</strong>: combines tiny slices under a single “Other” category to declutter charts.</li>
<li><strong>Exclude “-”</strong>: hides uncached entries (cache status “-”) from cache status distributions.</li>
</ul>
</div>
<div id="reports-breakdown"></div>
<div id="domain-section" class="is-hidden">
<div id="reports-domain"></div>
</div>
<div id="analysis-section" class="is-hidden">
@ -129,37 +79,23 @@
registerChart,
reset,
currentLoad,
sliceWindow,
excludeValues,
toPercent,
groupOthers,
movingAverage,
} from './chartManager.js';
const STATE_KEY = 'ngxstat-state-v2';
const intervalSelect = document.getElementById('interval-select');
const domainSelect = document.getElementById('domain-select');
const intervalControl = document.getElementById('interval-control');
const domainControl = document.getElementById('domain-control');
const timeControl = document.getElementById('time-control');
const timeSelect = document.getElementById('time-select');
const modePercentControl = document.getElementById('mode-percent-control');
const modeGroupControl = document.getElementById('mode-group-control');
const excludeUncachedControl = document.getElementById('exclude-uncached-control');
const smoothControl = document.getElementById('smooth-control');
const resetButton = document.getElementById('reset-view');
const tabs = document.querySelectorAll('#report-tabs li');
const sections = {
recent: document.getElementById('recent-section'),
trends: document.getElementById('trends-section'),
breakdown: document.getElementById('breakdown-section'),
overview: document.getElementById('overview-section'),
all: document.getElementById('all-section'),
domain: document.getElementById('domain-section'),
analysis: document.getElementById('analysis-section')
};
const containers = {
recent: document.getElementById('overview-reports'),
trends: document.getElementById('reports-trends'),
breakdown: document.getElementById('reports-breakdown')
overview: document.getElementById('overview-reports'),
all: document.getElementById('reports-all'),
domain: document.getElementById('reports-domain')
};
const recentRow = document.getElementById('recent-row');
const analysisElems = {
missing: document.getElementById('analysis-missing'),
cache: document.getElementById('analysis-cache'),
@ -172,142 +108,9 @@
const generatedElem = document.getElementById('stat-generated');
const elapsedElem = document.getElementById('stat-elapsed');
// Extra controls
// Legacy window select kept for internal state only (not shown)
const windowSelect = document.getElementById('window-select');
// If legacy window select is not present in DOM, create a hidden one for code paths
// that still reference it.
(function ensureHiddenWindowSelect(){
if (!windowSelect) {
const hidden = document.createElement('select');
hidden.id = 'window-select';
hidden.classList.add('is-hidden');
// Supported values used by code
['1h','24h','7d','30d','12w','12m','all'].forEach(v => {
const o = document.createElement('option');
o.value = v; o.textContent = v;
hidden.appendChild(o);
});
document.body.appendChild(hidden);
}
})();
const percentToggle = document.getElementById('percent-toggle');
const groupToggle = document.getElementById('group-toggle');
const excludeUncachedToggle = document.getElementById('exclude-uncached-toggle');
const smoothToggle = document.getElementById('smooth-toggle');
let currentInterval = intervalSelect.value;
let currentDomain = domainSelect.value;
let currentTab = 'recent';
let currentWindow = windowSelect ? windowSelect.value : '7d'; // 1h, 24h, 7d, 30d, 12w, 12m, all
let modePercent = false;
let modeGroup = true;
let excludeUncached = true;
let smoothError = false;
let hadExplicitWindow = false; // URL or saved-state provided window
function saveState() {
try {
localStorage.setItem(STATE_KEY, JSON.stringify({
tab: currentTab,
interval: currentInterval,
domain: currentDomain,
window: currentWindow,
percent: modePercent ? 1 : 0,
group: modeGroup ? 1 : 0,
exclude_dash: excludeUncached ? 1 : 0,
smooth: smoothError ? 1 : 0,
}));
} catch {}
}
function loadSavedState() {
try {
const s = JSON.parse(localStorage.getItem(STATE_KEY) || '{}');
if (s.tab) currentTab = s.tab;
if (s.interval) currentInterval = s.interval;
if (s.domain !== undefined) currentDomain = s.domain;
if (s.window) { currentWindow = s.window; hadExplicitWindow = true; }
if (s.percent !== undefined) modePercent = !!Number(s.percent);
if (s.group !== undefined) modeGroup = !!Number(s.group);
if (s.exclude_dash !== undefined) excludeUncached = !!Number(s.exclude_dash);
if (s.smooth !== undefined) smoothError = !!Number(s.smooth);
} catch {}
}
function applyURLParams() {
const params = new URLSearchParams(location.search);
if (params.get('tab')) currentTab = params.get('tab');
if (params.get('interval')) currentInterval = params.get('interval');
if (params.get('domain') !== null) currentDomain = params.get('domain') || '';
if (params.get('window')) { currentWindow = params.get('window'); hadExplicitWindow = true; }
if (params.get('percent') !== null) modePercent = params.get('percent') === '1';
if (params.get('group') !== null) modeGroup = params.get('group') === '1';
if (params.get('exclude_dash') !== null) excludeUncached = params.get('exclude_dash') === '1';
if (params.get('smooth') !== null) smoothError = params.get('smooth') === '1';
}
function updateURL() {
const params = new URLSearchParams();
params.set('tab', currentTab);
params.set('interval', currentInterval);
if (currentDomain) params.set('domain', currentDomain);
params.set('window', currentWindow);
params.set('percent', modePercent ? '1' : '0');
params.set('group', modeGroup ? '1' : '0');
params.set('exclude_dash', excludeUncached ? '1' : '0');
params.set('smooth', smoothError ? '1' : '0');
const newUrl = `${location.pathname}?${params.toString()}`;
history.replaceState(null, '', newUrl);
saveState();
}
function bucketsForWindow(win, interval) {
switch (win) {
case '1h': return interval === 'hourly' ? 1 : 'all';
case '24h': return interval === 'hourly' ? 24 : 'all';
case '7d': return interval === 'daily' ? 7 : 'all';
case '30d': return interval === 'daily' ? 30 : 'all';
case '12w': return interval === 'weekly' ? 12 : 'all';
case '12m': return interval === 'monthly' ? 12 : 'all';
default: return 'all';
}
}
function availableIntervals() {
try {
return Array.from(intervalSelect ? intervalSelect.options : []).map(o => o.value);
} catch { return []; }
}
function pickIntervalForWindow(win) {
const avail = availableIntervals();
const pref = (list) => list.find(x => avail.includes(x));
switch (win) {
case '1h':
case '24h':
return pref(['hourly','daily','weekly','monthly']) || (avail[0] || 'daily');
case '7d':
case '30d':
return pref(['daily','weekly','monthly','hourly']) || (avail[0] || 'daily');
case '12w':
return pref(['weekly','daily','monthly']) || (avail[0] || 'weekly');
case '12m':
return pref(['monthly','weekly','daily']) || (avail[0] || 'monthly');
default:
// all time: favor coarser buckets if available
return pref(['monthly','weekly','daily','hourly']) || (avail[0] || 'weekly');
}
}
function applyTimePreset(win) {
currentWindow = win;
currentInterval = pickIntervalForWindow(win);
if (intervalSelect) intervalSelect.value = currentInterval;
const winSel = document.getElementById('window-select');
if (winSel) winSel.value = currentWindow;
}
let currentTab = 'overview';
function initReport(token, rep, base) {
fetch(base + '/' + rep.json, { signal: token.controller.signal })
@ -330,69 +133,15 @@
return;
}
// Transform pipeline (client-only)
let transformed = data.slice();
const bucketField = bucketFields[0];
const isTimeSeries = bucketField === 'time_bucket';
// Exclusions (per-report) and explicit uncached toggle for cache_status
if (rep.exclude_values && rep.exclude_values.length) {
transformed = excludeValues(transformed, bucketField, rep.exclude_values);
}
if (excludeUncached && bucketField === 'cache_status') {
transformed = excludeValues(transformed, bucketField, ['-']);
}
// Windowing for time series
if (isTimeSeries) {
// Only apply windowing if report supports current window (if constrained)
const supported = Array.isArray(rep.windows_supported) ? rep.windows_supported : null;
const canWindow = !supported || supported.includes(currentWindow);
if (canWindow) {
const n = bucketsForWindow(currentWindow, currentInterval);
transformed = sliceWindow(transformed, n);
}
}
// Distributions: percent + group small
const isDistribution = ['pie', 'polarArea', 'doughnut', 'donut'].includes(rep.chart);
if (isDistribution) {
if (modeGroup) {
const thr = (typeof rep.group_others_threshold === 'number') ? rep.group_others_threshold : 0.03;
transformed = groupOthers(transformed, bucketField, 'value', thr, 'Other');
}
if (modePercent) {
transformed = toPercent(transformed, 'value');
}
}
// Relabel '-' to 'Uncached' for cache_status distributions
if (bucketField === 'cache_status') {
transformed = transformed.map(row => ({
...row,
[bucketField]: row[bucketField] === '-' ? 'Uncached' : row[bucketField]
}));
}
const labelsArr = transformed.map(x => x[bucketField]);
let values = transformed.map(x => x.value);
const labelsArr = data.map(x => x[bucketField]);
const values = data.map(x => x.value);
const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart;
const options = { scales: { y: { beginAtZero: true } } };
let datasets = [];
if (rep.chart === 'stackedBar') {
options.scales.x = { stacked: true };
options.scales.y = options.scales.y || {};
options.scales.y.stacked = true;
// Build multiple series from columns (exclude bucket & total)
const keys = transformed.length ? Object.keys(transformed[0]).filter(k => k !== bucketField && k !== 'total') : [];
const palette = rep.colors || rep.palette || [
'#3273dc', '#23d160', '#ffdd57', '#ff3860', '#7957d5', '#363636'
];
datasets = keys.map((k, i) => ({
label: k,
data: transformed.map(r => Number(r[k]) || 0),
backgroundColor: palette[i % palette.length],
borderColor: palette[i % palette.length],
borderWidth: 1,
fill: false,
}));
} else {
}
const dataset = {
label: rep.label,
data: values,
@ -402,9 +151,6 @@
if (rep.colors) {
dataset.backgroundColor = rep.colors;
dataset.borderColor = rep.colors;
} else if (rep.palette) {
dataset.backgroundColor = rep.palette;
dataset.borderColor = rep.palette;
} else if (rep.color) {
dataset.backgroundColor = rep.color;
dataset.borderColor = rep.color;
@ -412,18 +158,11 @@
dataset.backgroundColor = 'rgba(54, 162, 235, 0.5)';
dataset.borderColor = 'rgba(54, 162, 235, 1)';
}
// Optional smoothing for error_rate
if (rep.name === 'error_rate' && smoothError) {
dataset.data = movingAverage(values, 3);
dataset.label = rep.label + ' (smoothed)';
}
datasets = [dataset];
}
const chart = new Chart(document.getElementById('chart-' + rep.name), {
type: chartType,
data: {
labels: labelsArr,
datasets
datasets: [dataset]
},
options: options
});
@ -449,16 +188,21 @@
function loadReports() {
let path;
let container = containers[currentTab];
if (currentTab === 'recent') {
let container;
if (currentTab === 'overview') {
path = 'global';
container = containers.overview;
} else if (currentTab === 'all') {
path = currentInterval;
container = containers.all;
} else {
path = currentDomain ? ('domains/' + encodeURIComponent(currentDomain) + '/' + currentInterval) : currentInterval;
container = containers.domain;
if (!currentDomain) {
reset(container);
container.innerHTML = '<p>Select a domain</p>';
return;
}
// Clear the top row on each load of Recent
if (currentTab === 'recent' && recentRow) {
recentRow.innerHTML = '';
path = 'domains/' + encodeURIComponent(currentDomain) + '/' + currentInterval;
}
const token = newLoad(container);
@ -467,36 +211,12 @@
.then(r => r.json())
.then(reports => {
if (token !== currentLoad) return;
const isDistributionType = t => ['pie','polarArea','doughnut','donut'].includes(t);
const filtered = reports.filter(rep => {
if (currentTab === 'recent') return true;
if (currentTab === 'trends') return rep.chart !== 'table' && !isDistributionType(rep.chart);
if (currentTab === 'breakdown') return isDistributionType(rep.chart) || rep.chart === 'table';
return true;
});
// If no explicit window was given (URL or saved state), honor first report's default
if (!hadExplicitWindow) {
const withDefault = filtered.find(r => r.window_default);
if (withDefault && typeof withDefault.window_default === 'string') {
currentWindow = withDefault.window_default;
windowSelect.value = currentWindow;
updateURL();
}
}
filtered.forEach(rep => {
reports.forEach(rep => {
fetch(path + '/' + rep.html, { signal: token.controller.signal })
.then(r => r.text())
.then(html => {
if (token !== currentLoad) return;
// On Recent tab, render Cache Status and HTTP Statuses side-by-side
const inTopRow = currentTab === 'recent' &&
(rep.name === 'cache_status_breakdown' || rep.name === 'status_distribution');
if (inTopRow && recentRow) {
const wrapped = `<div class="column is-half">${html}</div>`;
recentRow.insertAdjacentHTML('beforeend', wrapped);
} else {
container.insertAdjacentHTML('beforeend', html);
}
initReport(token, rep, path);
});
});
@ -593,20 +313,9 @@
Object.entries(sections).forEach(([key, section]) => {
section.classList.toggle('is-hidden', key !== name);
});
const showTime = name !== 'recent' && name !== 'analysis';
const showDomain = showTime;
// Always keep legacy interval control hidden; use unified time control
intervalControl.classList.add('is-hidden');
domainControl.classList.toggle('is-hidden', !showDomain);
timeControl.classList.toggle('is-hidden', !showTime);
// Only show percent/group/exclude toggles on Breakdown tab,
// and smoothing only on Trends tab
modePercentControl.classList.toggle('is-hidden', name !== 'breakdown');
modeGroupControl.classList.toggle('is-hidden', name !== 'breakdown');
excludeUncachedControl.classList.toggle('is-hidden', name !== 'breakdown');
smoothControl.classList.toggle('is-hidden', name !== 'trends');
updateURL();
if (name === 'recent') {
intervalControl.classList.toggle('is-hidden', name === 'overview' || name === 'analysis');
domainControl.classList.toggle('is-hidden', name !== 'domain');
if (name === 'overview') {
loadStats();
}
if (name === 'analysis') {
@ -616,103 +325,26 @@
}
}
if (intervalSelect) {
intervalSelect.addEventListener('change', () => {
currentInterval = intervalSelect.value;
abortLoad(currentLoad);
Object.values(containers).forEach(reset);
updateURL();
reset(containers.all);
reset(containers.domain);
loadReports();
});
}
domainSelect.addEventListener('change', () => {
currentDomain = domainSelect.value;
abortLoad(currentLoad);
Object.values(containers).forEach(reset);
updateURL();
loadReports();
});
if (timeSelect) {
timeSelect.addEventListener('change', () => {
applyTimePreset(timeSelect.value);
abortLoad(currentLoad);
updateURL();
loadReports();
});
}
percentToggle.addEventListener('change', () => {
modePercent = percentToggle.checked;
abortLoad(currentLoad);
updateURL();
loadReports();
});
groupToggle.addEventListener('change', () => {
modeGroup = groupToggle.checked;
abortLoad(currentLoad);
updateURL();
loadReports();
});
excludeUncachedToggle.addEventListener('change', () => {
excludeUncached = excludeUncachedToggle.checked;
abortLoad(currentLoad);
updateURL();
loadReports();
});
smoothToggle.addEventListener('change', () => {
smoothError = smoothToggle.checked;
abortLoad(currentLoad);
updateURL();
reset(containers.domain);
loadReports();
});
tabs.forEach(tab => {
tab.addEventListener('click', () => {
switchTab(tab.dataset.tab);
tab.addEventListener('click', () => switchTab(tab.dataset.tab));
});
});
resetButton.addEventListener('click', () => {
try {
localStorage.removeItem('ngxstat-state'); // clear legacy
localStorage.removeItem(STATE_KEY);
} catch {}
// Reset to hard defaults
currentTab = 'recent';
currentInterval = intervalSelect ? (intervalSelect.value = intervalSelect.options[0]?.value || currentInterval) : currentInterval;
currentDomain = domainSelect.value = '';
applyTimePreset('7d');
if (timeSelect) timeSelect.value = '7d';
modePercent = percentToggle.checked = false;
modeGroup = groupToggle.checked = true;
excludeUncached = excludeUncachedToggle.checked = true;
smoothError = smoothToggle.checked = false;
hadExplicitWindow = false;
switchTab(currentTab);
});
// Initialize state (URL -> localStorage -> defaults)
loadSavedState();
applyURLParams();
// Sync controls
if (intervalSelect) intervalSelect.value = currentInterval;
domainSelect.value = currentDomain;
// Sync unified time select based on state
if (timeSelect) {
const known = new Set(['1h','24h','7d','30d','12w','12m','all']);
const pick = known.has(currentWindow) ? currentWindow : 'all';
timeSelect.value = pick;
applyTimePreset(pick);
}
percentToggle.checked = modePercent;
groupToggle.checked = modeGroup;
excludeUncachedToggle.checked = excludeUncached;
smoothToggle.checked = smoothError;
// Show/hide controls based on active tab
switchTab(currentTab);
switchTab('overview');
</script>
</body>
</html>

View file

@ -1,6 +1,12 @@
import sys
import json
import sqlite3
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(REPO_ROOT))
from scripts import analyze
from scripts import generate_reports as gr

View file

@ -1,3 +1,9 @@
import sys
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(REPO_ROOT))
from scripts import nginx_config as nc

View file

@ -1,10 +1,12 @@
import sqlite3
from pathlib import Path
import json
from datetime import datetime
import sys
import pytest
from typer.testing import CliRunner
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(REPO_ROOT))
from scripts import generate_reports as gr
@ -197,25 +199,9 @@ def test_generate_root_index(tmp_path, sample_reports, monkeypatch):
assert '<option value="Global">' not in content
assert '<option value="analysis">' not in content
def test_generated_marker_written(tmp_path, monkeypatch):
out_dir = tmp_path / "output"
monkeypatch.setattr(gr, "OUTPUT_DIR", out_dir)
monkeypatch.setattr(
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
)
monkeypatch.setattr(gr, "GENERATED_MARKER", out_dir / "generated.txt")
monkeypatch.setattr(gr, "_copy_icons", lambda: None)
(out_dir / "hourly").mkdir(parents=True)
runner = CliRunner()
result = runner.invoke(gr.app, ["index"])
assert result.exit_code == 0, result.output
marker = out_dir / "generated.txt"
assert marker.exists()
content = marker.read_text().strip()
datetime.strptime(content, "%Y-%m-%d %H:%M:%S")
# check for domain options
assert '<option value="foo.com">' in content
assert '<option value="bar.com">' in content
def test_global_reports_once(tmp_path, sample_reports, monkeypatch):
@ -323,93 +309,3 @@ def test_multi_bucket_table(tmp_path, monkeypatch):
entry = next(r for r in reports if r["name"] == "multi")
assert entry["buckets"] == ["domain", "agent"]
assert entry["bucket_label"] == ["Domain", "Agent"]
def test_top_n_limit_applied(tmp_path, monkeypatch):
# Prepare DB with many distinct agents
db_path = tmp_path / "database" / "ngxstat.db"
setup_db(db_path)
conn = sqlite3.connect(db_path)
cur = conn.cursor()
for i in range(10):
cur.execute(
"INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
"127.0.0.1",
"example.com",
f"2024-01-01 11:{i:02d}:00",
"GET /x HTTP/1.1",
200,
100,
"-",
f"ua-{i}",
"MISS",
),
)
conn.commit()
conn.close()
cfg = tmp_path / "reports.yml"
cfg.write_text(
"""
- name: agents
chart: table
global: true
top_n: 3
query: |
SELECT user_agent AS agent, COUNT(*) AS value
FROM logs
GROUP BY user_agent
ORDER BY value DESC
"""
)
monkeypatch.setattr(gr, "DB_PATH", db_path)
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
monkeypatch.setattr(
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
)
gr._generate_global()
data = json.loads((tmp_path / "output" / "global" / "agents.json").read_text())
# Should be limited to 3 rows
assert len(data) <= 3
def test_metadata_passthrough(tmp_path, monkeypatch):
db_path = tmp_path / "database" / "ngxstat.db"
setup_db(db_path)
cfg = tmp_path / "reports.yml"
cfg.write_text(
"""
- name: custom_ts
label: Custom TS
chart: line
window_default: 24h
windows_supported: [1h, 24h, 7d]
palette: ["#111111", "#222222"]
query: |
SELECT {bucket} AS time_bucket, COUNT(*) AS value
FROM logs
GROUP BY time_bucket
ORDER BY time_bucket
"""
)
monkeypatch.setattr(gr, "DB_PATH", db_path)
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
monkeypatch.setattr(
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
)
gr._generate_interval("hourly")
reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text())
entry = next(r for r in reports if r["name"] == "custom_ts")
assert entry["window_default"] == "24h"
assert entry["windows_supported"] == ["1h", "24h", "7d"]
assert entry["palette"] == ["#111111", "#222222"]

View file

@ -16,7 +16,7 @@ def test_script_invokes_commands(tmp_path):
python_stub = tmp_path / "python"
python_stub.write_text(f'#!/usr/bin/env bash\necho "$*" >> "{calls}"\n')
python_stub.chmod(0o755)
(tmp_path / "python3").write_text("#!/usr/bin/env bash\nexit 0\n")
(tmp_path / "python3").write_text(f"#!/usr/bin/env bash\nexit 0\n")
(tmp_path / "python3").chmod(0o755)
env = os.environ.copy()