UX Phase 1 follow-ups: state v2 + reset, window defaults + support, palette support; analysis JSON generation; tests for LIMIT/metadata; README updates
This commit is contained in:
ngxstat-bot
parent
fab91d2e04
commit
6de85b7cc5
5 changed files with 193 additions and 16 deletions
17
README.md
17
README.md
|
|
@ -39,9 +39,10 @@ all intervals in one go:
|
|||
```
|
||||
|
||||
The script calls `scripts/generate_reports.py` internally to create hourly,
|
||||
daily, weekly and monthly reports. Per-domain reports are written under
|
||||
`output/domains/<domain>` alongside the aggregate data. Open
|
||||
`output/index.html` in a browser to view the dashboard.
|
||||
daily, weekly and monthly reports, then writes analysis JSON files used by the
|
||||
"Analysis" tab. Per-domain reports are written under `output/domains/<domain>`
|
||||
alongside the aggregate data. Open `output/index.html` in a browser to view the
|
||||
dashboard.
|
||||
|
||||
If you prefer to run individual commands you can invoke the generator directly:
|
||||
|
||||
|
|
@ -54,8 +55,14 @@ python scripts/generate_reports.py daily --all-domains
|
|||
|
||||
`run-analysis.sh` executes additional utilities that examine the database for
|
||||
missing domains, caching opportunities and potential threats. The JSON output is
|
||||
saved under `output/analysis` and appears in the "Analysis" tab of the
|
||||
dashboard.
|
||||
saved under `output/analysis` and appears in the "Analysis" tab. The
|
||||
`run-reports.sh` script also generates these JSON files as part of the build.
|
||||
|
||||
## UX Controls
|
||||
|
||||
The dashboard defaults to a 7‑day window for time series. Your view preferences
|
||||
persist locally in the browser under the `ngxstat-state-v2` key. Use the
|
||||
"Reset view" button to clear saved state and restore defaults.
|
||||
|
||||
```bash
|
||||
./run-analysis.sh
|
||||
|
|
|
|||
|
|
@ -42,6 +42,10 @@ python scripts/generate_reports.py daily --all-domains
|
|||
python scripts/generate_reports.py weekly --all-domains
|
||||
python scripts/generate_reports.py monthly --all-domains
|
||||
|
||||
# Generate analysis JSON
|
||||
echo "[INFO] Generating analysis files..."
|
||||
python scripts/generate_reports.py analysis
|
||||
|
||||
# Generate root index
|
||||
python scripts/generate_reports.py index
|
||||
|
||||
|
|
|
|||
|
|
@ -344,6 +344,34 @@ def _generate_global() -> None:
|
|||
typer.echo("Generated global reports")
|
||||
|
||||
|
||||
def _generate_analysis() -> None:
|
||||
"""Generate analysis JSON files consumed by the Analysis tab."""
|
||||
try:
|
||||
# Import lazily to avoid circulars and keep dependencies optional
|
||||
from scripts import analyze
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
typer.echo(f"Failed to import analysis module: {exc}")
|
||||
return
|
||||
|
||||
# Ensure output root and icons present for parity
|
||||
_copy_icons()
|
||||
|
||||
# These commands write JSON files under output/analysis/
|
||||
try:
|
||||
analyze.check_missing_domains(json_output=True)
|
||||
except Exception as exc: # pragma: no cover - continue best-effort
|
||||
typer.echo(f"check_missing_domains failed: {exc}")
|
||||
try:
|
||||
analyze.suggest_cache(json_output=True)
|
||||
except Exception as exc: # pragma: no cover
|
||||
typer.echo(f"suggest_cache failed: {exc}")
|
||||
try:
|
||||
analyze.detect_threats()
|
||||
except Exception as exc: # pragma: no cover
|
||||
typer.echo(f"detect_threats failed: {exc}")
|
||||
typer.echo("Generated analysis JSON files")
|
||||
|
||||
|
||||
@app.command()
|
||||
def hourly(
|
||||
domain: Optional[str] = typer.Option(
|
||||
|
|
@ -414,6 +442,12 @@ def global_reports() -> None:
|
|||
_generate_global()
|
||||
|
||||
|
||||
@app.command()
|
||||
def analysis() -> None:
|
||||
"""Generate analysis JSON files for the Analysis tab."""
|
||||
_generate_analysis()
|
||||
|
||||
|
||||
@app.command()
|
||||
def index() -> None:
|
||||
"""Generate the root index page linking all reports."""
|
||||
|
|
|
|||
|
|
@ -74,6 +74,9 @@
|
|||
<input type="checkbox" id="exclude-uncached-toggle" checked> Exclude “-”
|
||||
</label>
|
||||
</div>
|
||||
<div id="reset-control" class="control">
|
||||
<button id="reset-view" class="button is-small is-light">Reset view</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="recent-section">
|
||||
|
|
@ -122,6 +125,7 @@
|
|||
groupOthers,
|
||||
movingAverage,
|
||||
} from './chartManager.js';
|
||||
const STATE_KEY = 'ngxstat-state-v2';
|
||||
const intervalSelect = document.getElementById('interval-select');
|
||||
const domainSelect = document.getElementById('domain-select');
|
||||
const intervalControl = document.getElementById('interval-control');
|
||||
|
|
@ -131,6 +135,7 @@
|
|||
const modeGroupControl = document.getElementById('mode-group-control');
|
||||
const excludeUncachedControl = document.getElementById('exclude-uncached-control');
|
||||
const smoothControl = document.getElementById('smooth-control');
|
||||
const resetButton = document.getElementById('reset-view');
|
||||
const tabs = document.querySelectorAll('#report-tabs li');
|
||||
const sections = {
|
||||
recent: document.getElementById('recent-section'),
|
||||
|
|
@ -172,10 +177,11 @@
|
|||
let modeGroup = true;
|
||||
let excludeUncached = true;
|
||||
let smoothError = false;
|
||||
let hadExplicitWindow = false; // URL or saved-state provided window
|
||||
|
||||
function saveState() {
|
||||
try {
|
||||
localStorage.setItem('ngxstat-state', JSON.stringify({
|
||||
localStorage.setItem(STATE_KEY, JSON.stringify({
|
||||
tab: currentTab,
|
||||
interval: currentInterval,
|
||||
domain: currentDomain,
|
||||
|
|
@ -190,11 +196,11 @@
|
|||
|
||||
function loadSavedState() {
|
||||
try {
|
||||
const s = JSON.parse(localStorage.getItem('ngxstat-state') || '{}');
|
||||
const s = JSON.parse(localStorage.getItem(STATE_KEY) || '{}');
|
||||
if (s.tab) currentTab = s.tab;
|
||||
if (s.interval) currentInterval = s.interval;
|
||||
if (s.domain !== undefined) currentDomain = s.domain;
|
||||
if (s.window) currentWindow = s.window;
|
||||
if (s.window) { currentWindow = s.window; hadExplicitWindow = true; }
|
||||
if (s.percent !== undefined) modePercent = !!Number(s.percent);
|
||||
if (s.group !== undefined) modeGroup = !!Number(s.group);
|
||||
if (s.exclude_dash !== undefined) excludeUncached = !!Number(s.exclude_dash);
|
||||
|
|
@ -207,7 +213,7 @@
|
|||
if (params.get('tab')) currentTab = params.get('tab');
|
||||
if (params.get('interval')) currentInterval = params.get('interval');
|
||||
if (params.get('domain') !== null) currentDomain = params.get('domain') || '';
|
||||
if (params.get('window')) currentWindow = params.get('window');
|
||||
if (params.get('window')) { currentWindow = params.get('window'); hadExplicitWindow = true; }
|
||||
if (params.get('percent') !== null) modePercent = params.get('percent') === '1';
|
||||
if (params.get('group') !== null) modeGroup = params.get('group') === '1';
|
||||
if (params.get('exclude_dash') !== null) excludeUncached = params.get('exclude_dash') === '1';
|
||||
|
|
@ -273,9 +279,14 @@
|
|||
}
|
||||
// Windowing for time series
|
||||
if (isTimeSeries) {
|
||||
// Only apply windowing if report supports current window (if constrained)
|
||||
const supported = Array.isArray(rep.windows_supported) ? rep.windows_supported : null;
|
||||
const canWindow = !supported || supported.includes(currentWindow);
|
||||
if (canWindow) {
|
||||
const n = bucketsForWindow(currentWindow, currentInterval);
|
||||
transformed = sliceWindow(transformed, n);
|
||||
}
|
||||
}
|
||||
// Distributions: percent + group small
|
||||
const isDistribution = ['pie', 'polarArea', 'doughnut', 'donut'].includes(rep.chart);
|
||||
if (isDistribution) {
|
||||
|
|
@ -306,7 +317,7 @@
|
|||
options.scales.y.stacked = true;
|
||||
// Build multiple series from columns (exclude bucket & total)
|
||||
const keys = transformed.length ? Object.keys(transformed[0]).filter(k => k !== bucketField && k !== 'total') : [];
|
||||
const palette = rep.colors || [
|
||||
const palette = rep.colors || rep.palette || [
|
||||
'#3273dc', '#23d160', '#ffdd57', '#ff3860', '#7957d5', '#363636'
|
||||
];
|
||||
datasets = keys.map((k, i) => ({
|
||||
|
|
@ -327,6 +338,9 @@
|
|||
if (rep.colors) {
|
||||
dataset.backgroundColor = rep.colors;
|
||||
dataset.borderColor = rep.colors;
|
||||
} else if (rep.palette) {
|
||||
dataset.backgroundColor = rep.palette;
|
||||
dataset.borderColor = rep.palette;
|
||||
} else if (rep.color) {
|
||||
dataset.backgroundColor = rep.color;
|
||||
dataset.borderColor = rep.color;
|
||||
|
|
@ -392,6 +406,15 @@
|
|||
if (currentTab === 'tables') return rep.chart === 'table';
|
||||
return true;
|
||||
});
|
||||
// If no explicit window was given (URL or saved state), honor first report's default
|
||||
if (!hadExplicitWindow) {
|
||||
const withDefault = filtered.find(r => r.window_default);
|
||||
if (withDefault && typeof withDefault.window_default === 'string') {
|
||||
currentWindow = withDefault.window_default;
|
||||
windowSelect.value = currentWindow;
|
||||
updateURL();
|
||||
}
|
||||
}
|
||||
filtered.forEach(rep => {
|
||||
fetch(path + '/' + rep.html, { signal: token.controller.signal })
|
||||
.then(r => r.text())
|
||||
|
|
@ -499,10 +522,12 @@
|
|||
intervalControl.classList.toggle('is-hidden', !showInterval);
|
||||
domainControl.classList.toggle('is-hidden', !showDomain);
|
||||
windowControl.classList.toggle('is-hidden', !showInterval);
|
||||
modePercentControl.classList.toggle('is-hidden', !showInterval);
|
||||
modeGroupControl.classList.toggle('is-hidden', !showInterval);
|
||||
excludeUncachedControl.classList.toggle('is-hidden', !showInterval);
|
||||
smoothControl.classList.toggle('is-hidden', !showInterval);
|
||||
// Only show percent/group/exclude toggles on Distribution tab,
|
||||
// and smoothing only on Trends tab
|
||||
modePercentControl.classList.toggle('is-hidden', name !== 'distribution');
|
||||
modeGroupControl.classList.toggle('is-hidden', name !== 'distribution');
|
||||
excludeUncachedControl.classList.toggle('is-hidden', name !== 'distribution');
|
||||
smoothControl.classList.toggle('is-hidden', name !== 'trends');
|
||||
updateURL();
|
||||
if (name === 'recent') {
|
||||
loadStats();
|
||||
|
|
@ -570,6 +595,23 @@
|
|||
switchTab(tab.dataset.tab);
|
||||
});
|
||||
});
|
||||
resetButton.addEventListener('click', () => {
|
||||
try {
|
||||
localStorage.removeItem('ngxstat-state'); // clear legacy
|
||||
localStorage.removeItem(STATE_KEY);
|
||||
} catch {}
|
||||
// Reset to hard defaults
|
||||
currentTab = 'recent';
|
||||
currentInterval = intervalSelect.value = intervalSelect.options[0]?.value || currentInterval;
|
||||
currentDomain = domainSelect.value = '';
|
||||
currentWindow = windowSelect.value = '7d';
|
||||
modePercent = percentToggle.checked = false;
|
||||
modeGroup = groupToggle.checked = true;
|
||||
excludeUncached = excludeUncachedToggle.checked = true;
|
||||
smoothError = smoothToggle.checked = false;
|
||||
hadExplicitWindow = false;
|
||||
switchTab(currentTab);
|
||||
});
|
||||
// Initialize state (URL -> localStorage -> defaults)
|
||||
loadSavedState();
|
||||
applyURLParams();
|
||||
|
|
|
|||
|
|
@ -323,3 +323,93 @@ def test_multi_bucket_table(tmp_path, monkeypatch):
|
|||
entry = next(r for r in reports if r["name"] == "multi")
|
||||
assert entry["buckets"] == ["domain", "agent"]
|
||||
assert entry["bucket_label"] == ["Domain", "Agent"]
|
||||
|
||||
|
||||
def test_top_n_limit_applied(tmp_path, monkeypatch):
|
||||
# Prepare DB with many distinct agents
|
||||
db_path = tmp_path / "database" / "ngxstat.db"
|
||||
setup_db(db_path)
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
for i in range(10):
|
||||
cur.execute(
|
||||
"INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
"127.0.0.1",
|
||||
"example.com",
|
||||
f"2024-01-01 11:{i:02d}:00",
|
||||
"GET /x HTTP/1.1",
|
||||
200,
|
||||
100,
|
||||
"-",
|
||||
f"ua-{i}",
|
||||
"MISS",
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
cfg = tmp_path / "reports.yml"
|
||||
cfg.write_text(
|
||||
"""
|
||||
- name: agents
|
||||
chart: table
|
||||
global: true
|
||||
top_n: 3
|
||||
query: |
|
||||
SELECT user_agent AS agent, COUNT(*) AS value
|
||||
FROM logs
|
||||
GROUP BY user_agent
|
||||
ORDER BY value DESC
|
||||
"""
|
||||
)
|
||||
|
||||
monkeypatch.setattr(gr, "DB_PATH", db_path)
|
||||
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
|
||||
monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
|
||||
monkeypatch.setattr(
|
||||
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
|
||||
)
|
||||
|
||||
gr._generate_global()
|
||||
|
||||
data = json.loads((tmp_path / "output" / "global" / "agents.json").read_text())
|
||||
# Should be limited to 3 rows
|
||||
assert len(data) <= 3
|
||||
|
||||
|
||||
def test_metadata_passthrough(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "database" / "ngxstat.db"
|
||||
setup_db(db_path)
|
||||
|
||||
cfg = tmp_path / "reports.yml"
|
||||
cfg.write_text(
|
||||
"""
|
||||
- name: custom_ts
|
||||
label: Custom TS
|
||||
chart: line
|
||||
window_default: 24h
|
||||
windows_supported: [1h, 24h, 7d]
|
||||
palette: ["#111111", "#222222"]
|
||||
query: |
|
||||
SELECT {bucket} AS time_bucket, COUNT(*) AS value
|
||||
FROM logs
|
||||
GROUP BY time_bucket
|
||||
ORDER BY time_bucket
|
||||
"""
|
||||
)
|
||||
|
||||
monkeypatch.setattr(gr, "DB_PATH", db_path)
|
||||
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
|
||||
monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
|
||||
monkeypatch.setattr(
|
||||
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
|
||||
)
|
||||
|
||||
gr._generate_interval("hourly")
|
||||
|
||||
reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text())
|
||||
entry = next(r for r in reports if r["name"] == "custom_ts")
|
||||
assert entry["window_default"] == "24h"
|
||||
assert entry["windows_supported"] == ["1h", "24h", "7d"]
|
||||
assert entry["palette"] == ["#111111", "#222222"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue