UX Phase 1 follow-ups: state v2 + reset, window defaults + support, palette support; analysis JSON generation; tests for LIMIT/metadata; README updates

README.md

@@ -39,9 +39,10 @@ all intervals in one go:
 ```
 
 The script calls `scripts/generate_reports.py` internally to create hourly,
-daily, weekly and monthly reports. Per-domain reports are written under
+daily, weekly and monthly reports, then writes analysis JSON files used by the
-`output/domains/<domain>` alongside the aggregate data. Open
+"Analysis" tab. Per-domain reports are written under `output/domains/<domain>`
-`output/index.html` in a browser to view the dashboard.
+alongside the aggregate data. Open `output/index.html` in a browser to view the
+dashboard.
 
 If you prefer to run individual commands you can invoke the generator directly:
 
@@ -54,8 +55,14 @@ python scripts/generate_reports.py daily --all-domains
 
 `run-analysis.sh` executes additional utilities that examine the database for
 missing domains, caching opportunities and potential threats. The JSON output is
-saved under `output/analysis` and appears in the "Analysis" tab of the
+saved under `output/analysis` and appears in the "Analysis" tab. The
-dashboard.
+`run-reports.sh` script also generates these JSON files as part of the build.
+
+## UX Controls
+
+The dashboard defaults to a 7‑day window for time series. Your view preferences
+persist locally in the browser under the `ngxstat-state-v2` key. Use the
+"Reset view" button to clear saved state and restore defaults.
 
 ```bash
 ./run-analysis.sh

run-reports.sh

@@ -42,6 +42,10 @@ python scripts/generate_reports.py daily --all-domains
 python scripts/generate_reports.py weekly --all-domains
 python scripts/generate_reports.py monthly --all-domains
 
+# Generate analysis JSON
+echo "[INFO] Generating analysis files..."
+python scripts/generate_reports.py analysis
+
 # Generate root index
 python scripts/generate_reports.py index
 

scripts/generate_reports.py

@@ -344,6 +344,34 @@ def _generate_global() -> None:
     typer.echo("Generated global reports")
 
 
+def _generate_analysis() -> None:
+    """Generate analysis JSON files consumed by the Analysis tab."""
+    try:
+        # Import lazily to avoid circulars and keep dependencies optional
+        from scripts import analyze
+    except Exception as exc:  # pragma: no cover - defensive
+        typer.echo(f"Failed to import analysis module: {exc}")
+        return
+
+    # Ensure output root and icons present for parity
+    _copy_icons()
+
+    # These commands write JSON files under output/analysis/
+    try:
+        analyze.check_missing_domains(json_output=True)
+    except Exception as exc:  # pragma: no cover - continue best-effort
+        typer.echo(f"check_missing_domains failed: {exc}")
+    try:
+        analyze.suggest_cache(json_output=True)
+    except Exception as exc:  # pragma: no cover
+        typer.echo(f"suggest_cache failed: {exc}")
+    try:
+        analyze.detect_threats()
+    except Exception as exc:  # pragma: no cover
+        typer.echo(f"detect_threats failed: {exc}")
+    typer.echo("Generated analysis JSON files")
+
+
 @app.command()
 def hourly(
     domain: Optional[str] = typer.Option(
@@ -414,6 +442,12 @@ def global_reports() -> None:
     _generate_global()
 
 
+@app.command()
+def analysis() -> None:
+    """Generate analysis JSON files for the Analysis tab."""
+    _generate_analysis()
+
+
 @app.command()
 def index() -> None:
     """Generate the root index page linking all reports."""

templates/index.html

@@ -74,6 +74,9 @@
           <input type="checkbox" id="exclude-uncached-toggle" checked> Exclude “-”
         </label>
       </div>
+      <div id="reset-control" class="control">
+        <button id="reset-view" class="button is-small is-light">Reset view</button>
+      </div>
     </div>
 
     <div id="recent-section">
@@ -122,6 +125,7 @@
       groupOthers,
       movingAverage,
     } from './chartManager.js';
+    const STATE_KEY = 'ngxstat-state-v2';
     const intervalSelect = document.getElementById('interval-select');
     const domainSelect = document.getElementById('domain-select');
     const intervalControl = document.getElementById('interval-control');
@@ -131,6 +135,7 @@
     const modeGroupControl = document.getElementById('mode-group-control');
     const excludeUncachedControl = document.getElementById('exclude-uncached-control');
     const smoothControl = document.getElementById('smooth-control');
+    const resetButton = document.getElementById('reset-view');
     const tabs = document.querySelectorAll('#report-tabs li');
     const sections = {
       recent: document.getElementById('recent-section'),
@@ -172,10 +177,11 @@
     let modeGroup = true;
     let excludeUncached = true;
     let smoothError = false;
+    let hadExplicitWindow = false; // URL or saved-state provided window
 
     function saveState() {
       try {
-        localStorage.setItem('ngxstat-state', JSON.stringify({
+        localStorage.setItem(STATE_KEY, JSON.stringify({
           tab: currentTab,
           interval: currentInterval,
           domain: currentDomain,
@@ -190,11 +196,11 @@
 
     function loadSavedState() {
       try {
-        const s = JSON.parse(localStorage.getItem('ngxstat-state') || '{}');
+        const s = JSON.parse(localStorage.getItem(STATE_KEY) || '{}');
         if (s.tab) currentTab = s.tab;
         if (s.interval) currentInterval = s.interval;
         if (s.domain !== undefined) currentDomain = s.domain;
-        if (s.window) currentWindow = s.window;
+        if (s.window) { currentWindow = s.window; hadExplicitWindow = true; }
         if (s.percent !== undefined) modePercent = !!Number(s.percent);
         if (s.group !== undefined) modeGroup = !!Number(s.group);
         if (s.exclude_dash !== undefined) excludeUncached = !!Number(s.exclude_dash);
@@ -207,7 +213,7 @@
       if (params.get('tab')) currentTab = params.get('tab');
       if (params.get('interval')) currentInterval = params.get('interval');
       if (params.get('domain') !== null) currentDomain = params.get('domain') || '';
-      if (params.get('window')) currentWindow = params.get('window');
+      if (params.get('window')) { currentWindow = params.get('window'); hadExplicitWindow = true; }
       if (params.get('percent') !== null) modePercent = params.get('percent') === '1';
       if (params.get('group') !== null) modeGroup = params.get('group') === '1';
       if (params.get('exclude_dash') !== null) excludeUncached = params.get('exclude_dash') === '1';
@@ -273,8 +279,13 @@
           }
           // Windowing for time series
           if (isTimeSeries) {
-            const n = bucketsForWindow(currentWindow, currentInterval);
+            // Only apply windowing if report supports current window (if constrained)
-            transformed = sliceWindow(transformed, n);
+            const supported = Array.isArray(rep.windows_supported) ? rep.windows_supported : null;
+            const canWindow = !supported || supported.includes(currentWindow);
+            if (canWindow) {
+              const n = bucketsForWindow(currentWindow, currentInterval);
+              transformed = sliceWindow(transformed, n);
+            }
           }
           // Distributions: percent + group small
           const isDistribution = ['pie', 'polarArea', 'doughnut', 'donut'].includes(rep.chart);
@@ -306,7 +317,7 @@
             options.scales.y.stacked = true;
             // Build multiple series from columns (exclude bucket & total)
             const keys = transformed.length ? Object.keys(transformed[0]).filter(k => k !== bucketField && k !== 'total') : [];
-            const palette = rep.colors || [
+            const palette = rep.colors || rep.palette || [
               '#3273dc', '#23d160', '#ffdd57', '#ff3860', '#7957d5', '#363636'
             ];
             datasets = keys.map((k, i) => ({
@@ -327,6 +338,9 @@
             if (rep.colors) {
               dataset.backgroundColor = rep.colors;
               dataset.borderColor = rep.colors;
+            } else if (rep.palette) {
+              dataset.backgroundColor = rep.palette;
+              dataset.borderColor = rep.palette;
             } else if (rep.color) {
               dataset.backgroundColor = rep.color;
               dataset.borderColor = rep.color;
@@ -392,6 +406,15 @@
             if (currentTab === 'tables') return rep.chart === 'table';
             return true;
           });
+          // If no explicit window was given (URL or saved state), honor first report's default
+          if (!hadExplicitWindow) {
+            const withDefault = filtered.find(r => r.window_default);
+            if (withDefault && typeof withDefault.window_default === 'string') {
+              currentWindow = withDefault.window_default;
+              windowSelect.value = currentWindow;
+              updateURL();
+            }
+          }
           filtered.forEach(rep => {
             fetch(path + '/' + rep.html, { signal: token.controller.signal })
               .then(r => r.text())
@@ -499,10 +522,12 @@
       intervalControl.classList.toggle('is-hidden', !showInterval);
       domainControl.classList.toggle('is-hidden', !showDomain);
       windowControl.classList.toggle('is-hidden', !showInterval);
-      modePercentControl.classList.toggle('is-hidden', !showInterval);
+      // Only show percent/group/exclude toggles on Distribution tab,
-      modeGroupControl.classList.toggle('is-hidden', !showInterval);
+      // and smoothing only on Trends tab
-      excludeUncachedControl.classList.toggle('is-hidden', !showInterval);
+      modePercentControl.classList.toggle('is-hidden', name !== 'distribution');
-      smoothControl.classList.toggle('is-hidden', !showInterval);
+      modeGroupControl.classList.toggle('is-hidden', name !== 'distribution');
+      excludeUncachedControl.classList.toggle('is-hidden', name !== 'distribution');
+      smoothControl.classList.toggle('is-hidden', name !== 'trends');
       updateURL();
       if (name === 'recent') {
         loadStats();
@@ -570,6 +595,23 @@
         switchTab(tab.dataset.tab);
       });
     });
+    resetButton.addEventListener('click', () => {
+      try {
+        localStorage.removeItem('ngxstat-state'); // clear legacy
+        localStorage.removeItem(STATE_KEY);
+      } catch {}
+      // Reset to hard defaults
+      currentTab = 'recent';
+      currentInterval = intervalSelect.value = intervalSelect.options[0]?.value || currentInterval;
+      currentDomain = domainSelect.value = '';
+      currentWindow = windowSelect.value = '7d';
+      modePercent = percentToggle.checked = false;
+      modeGroup = groupToggle.checked = true;
+      excludeUncached = excludeUncachedToggle.checked = true;
+      smoothError = smoothToggle.checked = false;
+      hadExplicitWindow = false;
+      switchTab(currentTab);
+    });
     // Initialize state (URL -> localStorage -> defaults)
     loadSavedState();
     applyURLParams();

tests/test_reports.py

@@ -323,3 +323,93 @@ def test_multi_bucket_table(tmp_path, monkeypatch):
     entry = next(r for r in reports if r["name"] == "multi")
     assert entry["buckets"] == ["domain", "agent"]
     assert entry["bucket_label"] == ["Domain", "Agent"]
+
+
+def test_top_n_limit_applied(tmp_path, monkeypatch):
+    # Prepare DB with many distinct agents
+    db_path = tmp_path / "database" / "ngxstat.db"
+    setup_db(db_path)
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+    for i in range(10):
+        cur.execute(
+            "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (
+                "127.0.0.1",
+                "example.com",
+                f"2024-01-01 11:{i:02d}:00",
+                "GET /x HTTP/1.1",
+                200,
+                100,
+                "-",
+                f"ua-{i}",
+                "MISS",
+            ),
+        )
+    conn.commit()
+    conn.close()
+
+    cfg = tmp_path / "reports.yml"
+    cfg.write_text(
+        """
+- name: agents
+  chart: table
+  global: true
+  top_n: 3
+  query: |
+    SELECT user_agent AS agent, COUNT(*) AS value
+    FROM logs
+    GROUP BY user_agent
+    ORDER BY value DESC
+"""
+    )
+
+    monkeypatch.setattr(gr, "DB_PATH", db_path)
+    monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
+    monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
+    monkeypatch.setattr(
+        gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
+    )
+
+    gr._generate_global()
+
+    data = json.loads((tmp_path / "output" / "global" / "agents.json").read_text())
+    # Should be limited to 3 rows
+    assert len(data) <= 3
+
+
+def test_metadata_passthrough(tmp_path, monkeypatch):
+    db_path = tmp_path / "database" / "ngxstat.db"
+    setup_db(db_path)
+
+    cfg = tmp_path / "reports.yml"
+    cfg.write_text(
+        """
+- name: custom_ts
+  label: Custom TS
+  chart: line
+  window_default: 24h
+  windows_supported: [1h, 24h, 7d]
+  palette: ["#111111", "#222222"]
+  query: |
+    SELECT {bucket} AS time_bucket, COUNT(*) AS value
+    FROM logs
+    GROUP BY time_bucket
+    ORDER BY time_bucket
+"""
+    )
+
+    monkeypatch.setattr(gr, "DB_PATH", db_path)
+    monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
+    monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
+    monkeypatch.setattr(
+        gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
+    )
+
+    gr._generate_interval("hourly")
+
+    reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text())
+    entry = next(r for r in reports if r["name"] == "custom_ts")
+    assert entry["window_default"] == "24h"
+    assert entry["windows_supported"] == ["1h", "24h", "7d"]
+    assert entry["palette"] == ["#111111", "#222222"]