analysis: make suggest_cache and detect_threats pure-callable, add CLI wrappers\n\n- Replace Typer Option defaults with plain Python defaults in functions used by generator/tests\n- Add CLI wrapper commands (, ) that delegate to the pure functions\n- Cast params to int for SQL/timedelta to avoid type issues\n- Resolves OptionInfo errors during run-reports analysis phase

- Replace tabs with Recent/Trends/Distribution/Tables/Analysis and add sticky controls (interval, domain, window [default 7d], percent, group small, exclude '-' -> Uncached, smoothing toggle).

- Client-side transforms: time-window slicing, percent mode, group others (3%), per-report exclusions; stackedBar multi-series; moving average for error_rate.

- Generator: pass through optional UX metadata (windows_supported, window_default, group_others_threshold, exclude_values, top_n, stacked, palette) and enforce top_n LIMIT for table reports.

- Reports: add status_classes_timeseries and cache_status_timeseries; apply top_n=50 to heavy tables.

- Chart manager: add helpers (sliceWindow, excludeValues, toPercent, groupOthers, movingAverage).

- URL state + localStorage for context; per-tab filtering for Trends/Distribution/Tables.

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+exclude = .git, .venv, output, static/icons
+max-line-length = 160

.forgejo/workflows/ci.yml

@@ -0,0 +1,151 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  ci:
+    name: Lint, test, and build
+    # This label must match your Forgejo runner's label
+    runs-on: docker
+    # Use a clean Debian container so tools are predictable
+    container: debian:stable-slim
+    env:
+      PYTHONDONTWRITEBYTECODE: "1"
+      PIP_DISABLE_PIP_VERSION_CHECK: "1"
+      UV_SYSTEM_PYTHON: "1"
+    steps:
+      - name: Install build tooling
+        run: |
+          set -euo pipefail
+          apt-get update
+          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+            git ca-certificates python3 python3-venv python3-pip python3-setuptools \
+            python3-wheel sqlite3
+          update-ca-certificates || true
+
+      - name: Checkout repository (manual)
+        run: |
+          set -euo pipefail
+          if [ -f Makefile ] || [ -d .git ]; then
+            echo "Repository present in workspace; skipping clone"
+            exit 0
+          fi
+          REMOTE_URL="${CI_REPOSITORY_URL:-}"
+          if [ -z "$REMOTE_URL" ]; then
+            if [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
+              REMOTE_URL="${GITHUB_SERVER_URL%/}/${GITHUB_REPOSITORY}.git"
+            elif [ -n "${GITHUB_REPOSITORY:-}" ]; then
+              REMOTE_URL="https://git.jordanwages.com/${GITHUB_REPOSITORY}.git"
+            else
+              echo "Unable to determine repository URL from CI environment" >&2
+              exit 1
+            fi
+          fi
+          AUTH_URL="$REMOTE_URL"
+          if [ -n "${GITHUB_TOKEN:-}" ]; then
+            ACTOR="${GITHUB_ACTOR:-oauth2}"
+            AUTH_URL=$(printf '%s' "$REMOTE_URL" | sed -E "s#^https://#https://${ACTOR}:${GITHUB_TOKEN}@#")
+          fi
+          echo "Cloning from: $REMOTE_URL"
+          if ! git clone --depth 1 "$AUTH_URL" .; then
+            echo "Auth clone failed; trying anonymous clone..." >&2
+            git clone --depth 1 "$REMOTE_URL" .
+          fi
+          if [ -n "${GITHUB_SHA:-}" ]; then
+            git fetch --depth 1 origin "$GITHUB_SHA" || true
+            git checkout -q "$GITHUB_SHA" || true
+          elif [ -n "${GITHUB_REF_NAME:-}" ]; then
+            git fetch --depth 1 origin "$GITHUB_REF_NAME" || true
+            git checkout -q "$GITHUB_REF_NAME" || true
+          fi
+
+      - name: Set up venv and install deps
+        run: |
+          set -euo pipefail
+          # Prefer persistent cache if runner provides /cache
+          USE_CACHE=0
+          if [ -d /cache ] && [ -w /cache ]; then
+            export PIP_CACHE_DIR=/cache/pip
+            mkdir -p "$PIP_CACHE_DIR"
+            REQ_HASH=$(sha256sum requirements.txt | awk '{print $1}')
+            PYVER=$(python3 -c 'import sys;print(".".join(map(str, sys.version_info[:2])))')
+            CACHE_VENV="/cache/venv-${REQ_HASH}-py${PYVER}"
+            if [ ! -f "$CACHE_VENV/bin/activate" ]; then
+              echo "Preparing cached virtualenv: $CACHE_VENV"
+              rm -rf "$CACHE_VENV" || true
+              python3 -m venv "$CACHE_VENV"
+            fi
+            ln -sfn "$CACHE_VENV" .venv
+            USE_CACHE=1
+          else
+            # Fallback to local venv
+            python3 -m venv .venv
+          fi
+
+          # If the link didn't produce an activate file, fallback to local venv
+          if [ ! -f .venv/bin/activate ]; then
+            echo "Cached venv missing; creating local .venv"
+            rm -f .venv
+            python3 -m venv .venv
+            USE_CACHE=0
+          fi
+
+          . .venv/bin/activate
+          python -m pip install --upgrade pip
+          if [ "$USE_CACHE" = "1" ]; then
+            # Ensure required packages are present; pip will use cache
+            pip install -r requirements.txt pytest || pip install -r requirements.txt pytest
+          else
+            pip install -r requirements.txt pytest
+          fi
+
+      - name: Format check (black)
+        run: |
+          . .venv/bin/activate
+          black --check .
+
+      - name: Lint (flake8)
+        run: |
+          . .venv/bin/activate
+          flake8 .
+
+      - name: Run tests (pytest)
+        run: |
+          . .venv/bin/activate
+          export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+          pytest -q --maxfail=1
+
+      - name: Build sample reports (no artifact upload)
+        run: |
+          set -euo pipefail
+          . .venv/bin/activate
+          python - <<'PY'
+          import sqlite3, pathlib
+          db = pathlib.Path('database/ngxstat.db')
+          db.parent.mkdir(parents=True, exist_ok=True)
+          conn = sqlite3.connect(db)
+          cur = conn.cursor()
+          cur.execute('''CREATE TABLE IF NOT EXISTS logs (
+              id INTEGER PRIMARY KEY,
+              ip TEXT,
+              host TEXT,
+              time TEXT,
+              request TEXT,
+              status INTEGER,
+              bytes_sent INTEGER,
+              referer TEXT,
+              user_agent TEXT,
+              cache_status TEXT
+          )''')
+          cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:00:00','GET / HTTP/1.1',200,100,'-','curl','MISS')")
+          cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:05:00','GET /about HTTP/1.1',200,100,'-','curl','MISS')")
+          conn.commit(); conn.close()
+          PY
+          python scripts/generate_reports.py global
+          python scripts/generate_reports.py hourly
+          python scripts/generate_reports.py index
+          tar -czf ngxstat-reports.tar.gz -C output .
+          echo "Built sample reports archive: ngxstat-reports.tar.gz"

README.md

@@ -39,9 +39,10 @@ all intervals in one go:
 ```
 
 The script calls `scripts/generate_reports.py` internally to create hourly,
-daily, weekly and monthly reports. Per-domain reports are written under
-`output/domains/<domain>` alongside the aggregate data. Open
-`output/index.html` in a browser to view the dashboard.
+daily, weekly and monthly reports, then writes analysis JSON files used by the
+"Analysis" tab. Per-domain reports are written under `output/domains/<domain>`
+alongside the aggregate data. Open `output/index.html` in a browser to view the
+dashboard.
 
 If you prefer to run individual commands you can invoke the generator directly:
 
@@ -54,8 +55,14 @@ python scripts/generate_reports.py daily --all-domains
 
 `run-analysis.sh` executes additional utilities that examine the database for
 missing domains, caching opportunities and potential threats. The JSON output is
-saved under `output/analysis` and appears in the "Analysis" tab of the
-dashboard.
+saved under `output/analysis` and appears in the "Analysis" tab. The
+`run-reports.sh` script also generates these JSON files as part of the build.
+
+## UX Controls
+
+The dashboard defaults to a 7‑day window for time series. Your view preferences
+persist locally in the browser under the `ngxstat-state-v2` key. Use the
+"Reset view" button to clear saved state and restore defaults.
 
 ```bash
 ./run-analysis.sh

reports.yml

@@ -48,6 +48,7 @@
   label: Top Domains
   icon: globe
   chart: table
+  top_n: 50
   per_domain: false
   bucket: domain
   bucket_label: Domain
@@ -75,47 +76,81 @@
   label: Top Paths
   icon: map
   chart: table
-  bucket: path
-  bucket_label: Path
+  top_n: 50
+  buckets:
+    - domain
+    - path
+  bucket_label:
+    - Domain
+    - Path
   query: |
-    SELECT path AS path,
-           COUNT(*) AS value
-    FROM (
-        SELECT substr(substr(request, instr(request, ' ') + 1), 1,
+    WITH paths AS (
+        SELECT host AS domain,
+               substr(substr(request, instr(request, ' ') + 1), 1,
                       instr(substr(request, instr(request, ' ') + 1), ' ') - 1) AS path
         FROM logs
+    ), ranked AS (
+        SELECT domain, path, COUNT(*) AS value,
+               ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn
+        FROM paths
+        GROUP BY domain, path
     )
-    GROUP BY path
-    ORDER BY value DESC
-    LIMIT 20
+    SELECT domain, path, value
+    FROM ranked
+    WHERE rn <= 20
+    ORDER BY domain, value DESC
 
 - name: user_agents
   label: User Agents
   icon: user
   chart: table
-  bucket: user_agent
-  bucket_label: User Agent
+  top_n: 50
+  buckets:
+    - domain
+    - user_agent
+  bucket_label:
+    - Domain
+    - User Agent
   query: |
-    SELECT user_agent AS user_agent,
-           COUNT(*) AS value
+    WITH ua AS (
+        SELECT host AS domain, user_agent
         FROM logs
-    GROUP BY user_agent
-    ORDER BY value DESC
-    LIMIT 20
+    ), ranked AS (
+        SELECT domain, user_agent, COUNT(*) AS value,
+               ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn
+        FROM ua
+        GROUP BY domain, user_agent
+    )
+    SELECT domain, user_agent, value
+    FROM ranked
+    WHERE rn <= 20
+    ORDER BY domain, value DESC
 
 - name: referrers
   label: Referrers
   icon: link
   chart: table
-  bucket: referrer
-  bucket_label: Referrer
+  top_n: 50
+  buckets:
+    - domain
+    - referrer
+  bucket_label:
+    - Domain
+    - Referrer
   query: |
-    SELECT referer AS referrer,
-           COUNT(*) AS value
+    WITH ref AS (
+        SELECT host AS domain, referer AS referrer
         FROM logs
-    GROUP BY referrer
-    ORDER BY value DESC
-    LIMIT 20
+    ), ranked AS (
+        SELECT domain, referrer, COUNT(*) AS value,
+               ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn
+        FROM ref
+        GROUP BY domain, referrer
+    )
+    SELECT domain, referrer, value
+    FROM ranked
+    WHERE rn <= 20
+    ORDER BY domain, value DESC
 
 - name: status_distribution
   label: HTTP Statuses
@@ -139,3 +174,40 @@
     - "#209cee"
     - "#ffdd57"
     - "#f14668"
+
+# New time-series: status classes over time (stacked)
+- name: status_classes_timeseries
+  label: Status Classes Over Time
+  icon: server
+  chart: stackedBar
+  bucket: time_bucket
+  bucket_label: Time
+  stacked: true
+  query: |
+    SELECT {bucket} AS time_bucket,
+           SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS "2xx",
+           SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END) AS "3xx",
+           SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS "4xx",
+           SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS "5xx",
+           COUNT(*) AS total
+    FROM logs
+    GROUP BY time_bucket
+    ORDER BY time_bucket
+
+# New time-series: cache status over time (compact Hit/Miss; exclude '-' by default)
+- name: cache_status_timeseries
+  label: Cache Status Over Time
+  icon: archive
+  chart: stackedBar
+  bucket: time_bucket
+  bucket_label: Time
+  stacked: true
+  exclude_values: ["-"]
+  query: |
+    SELECT {bucket} AS time_bucket,
+           SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) AS hit,
+           SUM(CASE WHEN cache_status = 'MISS' THEN 1 ELSE 0 END) AS miss,
+           COUNT(*) AS total
+    FROM logs
+    GROUP BY time_bucket
+    ORDER BY time_bucket

run-reports.sh

@@ -29,21 +29,25 @@ fi
 
 # Generate reports for all domains combined
 echo "[INFO] Generating aggregate reports..."
-python scripts/generate_reports.py hourly
-python scripts/generate_reports.py daily
-python scripts/generate_reports.py weekly
-python scripts/generate_reports.py monthly
-python scripts/generate_reports.py global
+python -m scripts.generate_reports hourly
+python -m scripts.generate_reports daily
+python -m scripts.generate_reports weekly
+python -m scripts.generate_reports monthly
+python -m scripts.generate_reports global
 
 # Generate reports for each individual domain
 echo "[INFO] Generating per-domain reports..."
-python scripts/generate_reports.py hourly --all-domains
-python scripts/generate_reports.py daily --all-domains
-python scripts/generate_reports.py weekly --all-domains
-python scripts/generate_reports.py monthly --all-domains
+python -m scripts.generate_reports hourly --all-domains
+python -m scripts.generate_reports daily --all-domains
+python -m scripts.generate_reports weekly --all-domains
+python -m scripts.generate_reports monthly --all-domains
+
+# Generate analysis JSON
+echo "[INFO] Generating analysis files..."
+python -m scripts.generate_reports analysis
 
 # Generate root index
-python scripts/generate_reports.py index
+python -m scripts.generate_reports index
 
 # Deactivate to keep cron environment clean
 if type deactivate >/dev/null 2>&1; then

scripts/analyze.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 
 import sqlite3
 from pathlib import Path
-from typing import Dict, List, Optional, Set
+from typing import List, Optional, Set
 from datetime import datetime, timedelta
 
 import json
@@ -155,10 +155,9 @@ def check_missing_domains(
             typer.echo(d)
 
 
-@app.command("suggest-cache")
 def suggest_cache(
-    threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"),
-    json_output: bool = typer.Option(False, "--json", help="Output results as JSON"),
+    threshold: int = 10,
+    json_output: bool = False,
 ) -> None:
     """Suggest domain/path pairs that could benefit from caching.
 
@@ -191,7 +190,7 @@ def suggest_cache(
         HAVING miss_count >= ?
         ORDER BY miss_count DESC
         """,
-        (threshold,),
+        (int(threshold),),
     )
 
     rows = [r for r in cur.fetchall() if r[0] in no_cache]
@@ -211,11 +210,18 @@ def suggest_cache(
         for item in result:
             typer.echo(f"{item['host']} {item['path']} {item['misses']}")
 
+@app.command("suggest-cache")
+def suggest_cache_cli(
+    threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"),
+    json_output: bool = typer.Option(False, "--json", help="Output results as JSON"),
+) -> None:
+    """CLI wrapper for suggest_cache."""
+    suggest_cache(threshold=threshold, json_output=json_output)
+
 
-@app.command("detect-threats")
 def detect_threats(
-    hours: int = typer.Option(1, help="Number of recent hours to analyze"),
-    ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"),
+    hours: int = 1,
+    ip_threshold: int = 100,
 ) -> None:
     """Detect potential security threats from recent logs."""
 
@@ -231,8 +237,8 @@ def detect_threats(
 
     max_dt = datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S")
     recent_end = max_dt
-    recent_start = recent_end - timedelta(hours=hours)
-    prev_start = recent_start - timedelta(hours=hours)
+    recent_start = recent_end - timedelta(hours=int(hours))
+    prev_start = recent_start - timedelta(hours=int(hours))
     prev_end = recent_start
 
     fmt = "%Y-%m-%d %H:%M:%S"
@@ -339,6 +345,14 @@ def detect_threats(
     out_path.write_text(json.dumps(report, indent=2))
     typer.echo(json.dumps(report))
 
+@app.command("detect-threats")
+def detect_threats_cli(
+    hours: int = typer.Option(1, help="Number of recent hours to analyze"),
+    ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"),
+) -> None:
+    """CLI wrapper for detect_threats."""
+    detect_threats(hours=hours, ip_threshold=ip_threshold)
+
 
 if __name__ == "__main__":
     app()

scripts/generate_reports.py

@@ -1,9 +1,10 @@
 import json
+import sys
 import sqlite3
 from pathlib import Path
 import shutil
 from typing import List, Dict, Optional
-from datetime import datetime
+from datetime import datetime, timezone
 import time
 
 import yaml
@@ -11,10 +12,16 @@ import yaml
 import typer
 from jinja2 import Environment, FileSystemLoader
 
+# Ensure project root is importable when running as a script (python scripts/generate_reports.py)
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
 DB_PATH = Path("database/ngxstat.db")
 OUTPUT_DIR = Path("output")
 TEMPLATE_DIR = Path("templates")
 REPORT_CONFIG = Path("reports.yml")
+GENERATED_MARKER = OUTPUT_DIR / "generated.txt"
 
 # Mapping of interval names to SQLite strftime formats.  These strings are
 # substituted into report queries whenever the special ``{bucket}`` token is
@@ -30,6 +37,19 @@ INTERVAL_FORMATS = {
 app = typer.Typer(help="Generate aggregated log reports")
 
 
+@app.callback()
+def _cli_callback(ctx: typer.Context) -> None:
+    """Register post-command hook to note generation time."""
+
+    def _write_marker() -> None:
+        OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+        # Use timezone-aware UTC to avoid deprecation warnings and ambiguity
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+        GENERATED_MARKER.write_text(f"{timestamp}\n")
+
+    ctx.call_on_close(_write_marker)
+
+
 def _get_domains() -> List[str]:
     """Return a sorted list of unique domains from the logs table."""
     conn = sqlite3.connect(DB_PATH)
@@ -58,15 +78,18 @@ def _save_json(path: Path, data: List[Dict]) -> None:
 
 
 def _copy_icons() -> None:
-    """Copy vendored icons to the output directory."""
+    """Copy vendored icons and scripts to the output directory."""
     src_dir = Path("static/icons")
     dst_dir = OUTPUT_DIR / "icons"
-    if not src_dir.is_dir():
-        return
+    if src_dir.is_dir():
         dst_dir.mkdir(parents=True, exist_ok=True)
         for icon in src_dir.glob("*.svg"):
             shutil.copy(icon, dst_dir / icon.name)
 
+    js_src = Path("static/chartManager.js")
+    if js_src.is_file():
+        shutil.copy(js_src, OUTPUT_DIR / js_src.name)
+
 
 def _render_snippet(report: Dict, out_dir: Path) -> None:
     """Render a single report snippet to ``<name>.html`` inside ``out_dir``."""
@@ -162,6 +185,16 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
         name = definition["name"]
         query = definition["query"].replace("{bucket}", bucket)
         query = query.replace("FROM logs", "FROM logs_view")
+        # Apply top_n limit for tables (performance-friendly), if configured
+        top_n = definition.get("top_n")
+        chart_type = definition.get("chart", "line")
+        if top_n and chart_type == "table":
+            try:
+                n = int(top_n)
+                if "LIMIT" not in query.upper():
+                    query = f"{query}\nLIMIT {n}"
+            except Exception:
+                pass
         cur.execute(query)
         rows = cur.fetchall()
         headers = [c[0] for c in cur.description]
@@ -179,12 +212,26 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
             entry["icon"] = definition["icon"]
         if "bucket" in definition:
             entry["bucket"] = definition["bucket"]
+        if "buckets" in definition:
+            entry["buckets"] = definition["buckets"]
         if "bucket_label" in definition:
             entry["bucket_label"] = definition["bucket_label"]
         if "color" in definition:
             entry["color"] = definition["color"]
         if "colors" in definition:
             entry["colors"] = definition["colors"]
+        # Optional UX metadata passthrough for frontend-only transforms
+        for key in (
+            "windows_supported",
+            "window_default",
+            "group_others_threshold",
+            "exclude_values",
+            "top_n",
+            "stacked",
+            "palette",
+        ):
+            if key in definition:
+                entry[key] = definition[key]
         _render_snippet(entry, out_dir)
         report_list.append(entry)
 
@@ -231,7 +278,8 @@ def _generate_global() -> None:
         return
 
     start_time = time.time()
-    generated_at = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+    # Use timezone-aware UTC for generated_at (string remains unchanged format)
+    generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
 
     _copy_icons()
 
@@ -248,6 +296,16 @@ def _generate_global() -> None:
 
         name = definition["name"]
         query = definition["query"]
+        # Apply top_n limit for tables (performance-friendly), if configured
+        top_n = definition.get("top_n")
+        chart_type = definition.get("chart", "line")
+        if top_n and chart_type == "table":
+            try:
+                n = int(top_n)
+                if "LIMIT" not in query.upper():
+                    query = f"{query}\nLIMIT {n}"
+            except Exception:
+                pass
         cur.execute(query)
         rows = cur.fetchall()
         headers = [c[0] for c in cur.description]
@@ -265,12 +323,26 @@ def _generate_global() -> None:
             entry["icon"] = definition["icon"]
         if "bucket" in definition:
             entry["bucket"] = definition["bucket"]
+        if "buckets" in definition:
+            entry["buckets"] = definition["buckets"]
         if "bucket_label" in definition:
             entry["bucket_label"] = definition["bucket_label"]
         if "color" in definition:
             entry["color"] = definition["color"]
         if "colors" in definition:
             entry["colors"] = definition["colors"]
+        # Optional UX metadata passthrough for frontend-only transforms
+        for key in (
+            "windows_supported",
+            "window_default",
+            "group_others_threshold",
+            "exclude_values",
+            "top_n",
+            "stacked",
+            "palette",
+        ):
+            if key in definition:
+                entry[key] = definition[key]
         _render_snippet(entry, out_dir)
         report_list.append(entry)
 
@@ -280,6 +352,34 @@ def _generate_global() -> None:
     typer.echo("Generated global reports")
 
 
+def _generate_analysis() -> None:
+    """Generate analysis JSON files consumed by the Analysis tab."""
+    try:
+        # Import lazily to avoid circulars and keep dependencies optional
+        from scripts import analyze
+    except Exception as exc:  # pragma: no cover - defensive
+        typer.echo(f"Failed to import analysis module: {exc}")
+        return
+
+    # Ensure output root and icons present for parity
+    _copy_icons()
+
+    # These commands write JSON files under output/analysis/
+    try:
+        analyze.check_missing_domains(json_output=True)
+    except Exception as exc:  # pragma: no cover - continue best-effort
+        typer.echo(f"check_missing_domains failed: {exc}")
+    try:
+        analyze.suggest_cache(json_output=True)
+    except Exception as exc:  # pragma: no cover
+        typer.echo(f"suggest_cache failed: {exc}")
+    try:
+        analyze.detect_threats()
+    except Exception as exc:  # pragma: no cover
+        typer.echo(f"detect_threats failed: {exc}")
+    typer.echo("Generated analysis JSON files")
+
+
 @app.command()
 def hourly(
     domain: Optional[str] = typer.Option(
@@ -350,6 +450,12 @@ def global_reports() -> None:
     _generate_global()
 
 
+@app.command()
+def analysis() -> None:
+    """Generate analysis JSON files for the Analysis tab."""
+    _generate_analysis()
+
+
 @app.command()
 def index() -> None:
     """Generate the root index page linking all reports."""

static/chartManager.js

@@ -0,0 +1,109 @@
+export let currentLoad = null;
+const loadInfo = new Map();
+
+export function newLoad(container) {
+  if (currentLoad) {
+    abortLoad(currentLoad);
+  }
+  reset(container);
+  const controller = new AbortController();
+  const token = { controller, charts: new Map() };
+  loadInfo.set(token, token);
+  currentLoad = token;
+  return token;
+}
+
+export function abortLoad(token) {
+  const info = loadInfo.get(token);
+  if (!info) return;
+  info.controller.abort();
+  info.charts.forEach(chart => {
+    try {
+      chart.destroy();
+    } catch (e) {}
+  });
+  loadInfo.delete(token);
+  if (currentLoad === token) {
+    currentLoad = null;
+  }
+}
+
+export function registerChart(token, id, chart) {
+  const info = loadInfo.get(token);
+  if (info) {
+    info.charts.set(id, chart);
+  } else {
+    chart.destroy();
+  }
+}
+
+export function reset(container) {
+  if (!container) return;
+  container.querySelectorAll('canvas').forEach(c => {
+    const chart = Chart.getChart(c);
+    if (chart) {
+      chart.destroy();
+    }
+  });
+  container.innerHTML = '';
+}
+
+// ---- Lightweight client-side data helpers ----
+
+// Slice last N rows from a time-ordered array
+export function sliceWindow(data, n) {
+  if (!Array.isArray(data) || n === undefined || n === null) return data;
+  if (n === 'all') return data;
+  const count = Number(n);
+  if (!Number.isFinite(count) || count <= 0) return data;
+  return data.slice(-count);
+}
+
+// Exclude rows whose value in key is in excluded list
+export function excludeValues(data, key, excluded = []) {
+  if (!excluded || excluded.length === 0) return data;
+  const set = new Set(excluded);
+  return data.filter(row => !set.has(row[key]));
+}
+
+// Compute percentages for categorical distributions (valueKey default 'value')
+export function toPercent(data, valueKey = 'value') {
+  const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0);
+  if (total <= 0) return data.map(r => ({ ...r }));
+  return data.map(r => ({ ...r, [valueKey]: (Number(r[valueKey]) || 0) * 100 / total }));
+}
+
+// Group categories with share < threshold into an 'Other' bucket.
+export function groupOthers(data, bucketKey, valueKey = 'value', threshold = 0.03, otherLabel = 'Other') {
+  if (!Array.isArray(data) || data.length === 0) return data;
+  const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0);
+  if (total <= 0) return data;
+  const major = [];
+  let other = 0;
+  for (const r of data) {
+    const v = Number(r[valueKey]) || 0;
+    if (total && v / total < threshold) {
+      other += v;
+    } else {
+      major.push({ ...r });
+    }
+  }
+  if (other > 0) major.push({ [bucketKey]: otherLabel, [valueKey]: other });
+  return major;
+}
+
+// Simple moving average over numeric array
+export function movingAverage(series, span = 3) {
+  const n = Math.max(1, Number(span) || 1);
+  const out = [];
+  for (let i = 0; i < series.length; i++) {
+    const start = Math.max(0, i - n + 1);
+    let sum = 0, cnt = 0;
+    for (let j = start; j <= i; j++) {
+      const v = Number(series[j]);
+      if (Number.isFinite(v)) { sum += v; cnt++; }
+    }
+    out.push(cnt ? sum / cnt : null);
+  }
+  return out;
+}

templates/index.html

@@ -12,14 +12,15 @@
 
     <div class="tabs is-toggle" id="report-tabs">
       <ul>
-        <li class="is-active" data-tab="overview"><a>Overview</a></li>
-        <li data-tab="all"><a>All Domains</a></li>
-        <li data-tab="domain"><a>Per Domain</a></li>
+        <li class="is-active" data-tab="recent"><a>Recent</a></li>
+        <li data-tab="trends"><a>Trends</a></li>
+        <li data-tab="breakdown"><a>Breakdown</a></li>
         <li data-tab="analysis"><a>Analysis</a></li>
       </ul>
     </div>
 
-    <div id="controls" class="field is-grouped mb-4">
+    <div id="controls" class="field is-grouped is-align-items-center mb-4" style="position: sticky; top: 0; background: white; z-index: 2; padding: 0.5rem 0;">
+      <!-- Hidden native interval control kept for compatibility and availability probing -->
       <div id="interval-control" class="control has-icons-left is-hidden">
         <div class="select is-small">
           <select id="interval-select">
@@ -41,26 +42,75 @@
         </div>
         <span class="icon is-small is-left"><img src="icons/server.svg" alt="Domain"></span>
       </div>
+      <!-- Unified Time control: selects both range and sensible grouping -->
+      <div id="time-control" class="control has-icons-left is-hidden">
+        <div class="select is-small">
+          <select id="time-select">
+            <option value="1h">Last hour</option>
+            <option value="24h">Last 24 hours</option>
+            <option value="7d" selected>Last 7 days</option>
+            <option value="30d">Last 30 days</option>
+            <option value="12w">Last 12 weeks</option>
+            <option value="12m">Last 12 months</option>
+            <option value="all">All time</option>
+          </select>
+        </div>
+        <span class="icon is-small is-left"><img src="icons/clock.svg" alt="Time"></span>
+      </div>
+      <div id="smooth-control" class="control is-hidden">
+        <label class="checkbox is-small">
+          <input type="checkbox" id="smooth-toggle"> Smooth error rate
+        </label>
+      </div>
+      <div id="mode-percent-control" class="control is-hidden">
+        <label class="checkbox is-small" title="Show values as a percentage of the total, instead of raw counts.">
+          <input type="checkbox" id="percent-toggle"> Percent mode
+        </label>
+      </div>
+      <div id="mode-group-control" class="control is-hidden">
+        <label class="checkbox is-small" title="Combine small categories into an 'Other' slice to declutter charts.">
+          <input type="checkbox" id="group-toggle" checked> Group small into Other
+        </label>
+      </div>
+      <div id="exclude-uncached-control" class="control is-hidden">
+        <label class="checkbox is-small" title="Hide uncached entries (cache status '-') from cache status distributions.">
+          <input type="checkbox" id="exclude-uncached-toggle" checked> Exclude “-”
+        </label>
+      </div>
+      <div id="reset-control" class="control">
+        <button id="reset-view" class="button is-small is-light">Reset view</button>
+      </div>
     </div>
 
-    <div id="overview-section">
+    <div id="recent-section">
       <div id="overview" class="box mb-5">
-        <h2 class="subtitle">Overview</h2>
+        <h2 class="subtitle">Recent</h2>
         <p>Total logs: <span id="stat-total">-</span></p>
         <p>Date range: <span id="stat-start">-</span> to <span id="stat-end">-</span></p>
         <p>Unique domains: <span id="stat-domains">-</span></p>
         <p>Last generated: <span id="stat-generated">-</span></p>
         <p>Generation time: <span id="stat-elapsed">-</span> seconds</p>
       </div>
+      <!-- Two key distributions side-by-side on Recent -->
+      <div id="recent-row" class="columns"></div>
       <div id="overview-reports"></div>
     </div>
 
-    <div id="all-section" class="is-hidden">
-      <div id="reports-all"></div>
+    <div id="trends-section" class="is-hidden">
+      <div id="reports-trends"></div>
     </div>
 
-  <div id="domain-section" class="is-hidden">
-    <div id="reports-domain"></div>
+    <div id="breakdown-section" class="is-hidden">
+      <div class="box mb-4">
+        <h2 class="subtitle">Breakdown</h2>
+        <p class="mb-2">Explore categorical distributions and detailed lists side-by-side. Use the options below to adjust how categories are shown.</p>
+        <ul style="margin-left: 1.2rem; list-style: disc;">
+          <li><strong>Percent mode</strong>: converts counts into percentages of the total for easier comparison.</li>
+          <li><strong>Group small into Other</strong>: combines tiny slices under a single “Other” category to declutter charts.</li>
+          <li><strong>Exclude “-”</strong>: hides uncached entries (cache status “-”) from cache status distributions.</li>
+        </ul>
+      </div>
+      <div id="reports-breakdown"></div>
     </div>
 
   <div id="analysis-section" class="is-hidden">
@@ -72,23 +122,44 @@
   <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
   <script src="https://cdn.jsdelivr.net/npm/jquery@3.7.0/dist/jquery.min.js"></script>
   <script src="https://cdn.datatables.net/1.13.4/js/jquery.dataTables.min.js"></script>
-  <script>
+  <script type="module">
+    import {
+      newLoad,
+      abortLoad,
+      registerChart,
+      reset,
+      currentLoad,
+      sliceWindow,
+      excludeValues,
+      toPercent,
+      groupOthers,
+      movingAverage,
+    } from './chartManager.js';
+    const STATE_KEY = 'ngxstat-state-v2';
     const intervalSelect = document.getElementById('interval-select');
     const domainSelect = document.getElementById('domain-select');
     const intervalControl = document.getElementById('interval-control');
     const domainControl = document.getElementById('domain-control');
+    const timeControl = document.getElementById('time-control');
+    const timeSelect = document.getElementById('time-select');
+    const modePercentControl = document.getElementById('mode-percent-control');
+    const modeGroupControl = document.getElementById('mode-group-control');
+    const excludeUncachedControl = document.getElementById('exclude-uncached-control');
+    const smoothControl = document.getElementById('smooth-control');
+    const resetButton = document.getElementById('reset-view');
     const tabs = document.querySelectorAll('#report-tabs li');
     const sections = {
-      overview: document.getElementById('overview-section'),
-      all: document.getElementById('all-section'),
-      domain: document.getElementById('domain-section'),
+      recent: document.getElementById('recent-section'),
+      trends: document.getElementById('trends-section'),
+      breakdown: document.getElementById('breakdown-section'),
       analysis: document.getElementById('analysis-section')
     };
     const containers = {
-      overview: document.getElementById('overview-reports'),
-      all: document.getElementById('reports-all'),
-      domain: document.getElementById('reports-domain')
+      recent: document.getElementById('overview-reports'),
+      trends: document.getElementById('reports-trends'),
+      breakdown: document.getElementById('reports-breakdown')
     };
+    const recentRow = document.getElementById('recent-row');
     const analysisElems = {
       missing: document.getElementById('analysis-missing'),
       cache: document.getElementById('analysis-cache'),
@@ -101,35 +172,227 @@
     const generatedElem = document.getElementById('stat-generated');
     const elapsedElem = document.getElementById('stat-elapsed');
 
+    // Extra controls
+    // Legacy window select kept for internal state only (not shown)
+    const windowSelect = document.getElementById('window-select');
+    
+    // If legacy window select is not present in DOM, create a hidden one for code paths
+    // that still reference it.
+    (function ensureHiddenWindowSelect(){
+      if (!windowSelect) {
+        const hidden = document.createElement('select');
+        hidden.id = 'window-select';
+        hidden.classList.add('is-hidden');
+        // Supported values used by code
+        ['1h','24h','7d','30d','12w','12m','all'].forEach(v => {
+          const o = document.createElement('option');
+          o.value = v; o.textContent = v;
+          hidden.appendChild(o);
+        });
+        document.body.appendChild(hidden);
+      }
+    })();
+    const percentToggle = document.getElementById('percent-toggle');
+    const groupToggle = document.getElementById('group-toggle');
+    const excludeUncachedToggle = document.getElementById('exclude-uncached-toggle');
+    const smoothToggle = document.getElementById('smooth-toggle');
+
     let currentInterval = intervalSelect.value;
     let currentDomain = domainSelect.value;
-    let currentTab = 'overview';
+    let currentTab = 'recent';
+    let currentWindow = windowSelect ? windowSelect.value : '7d'; // 1h, 24h, 7d, 30d, 12w, 12m, all
+    let modePercent = false;
+    let modeGroup = true;
+    let excludeUncached = true;
+    let smoothError = false;
+    let hadExplicitWindow = false; // URL or saved-state provided window
 
-    function initReport(rep, base) {
-      fetch(base + '/' + rep.json)
+    function saveState() {
+      try {
+        localStorage.setItem(STATE_KEY, JSON.stringify({
+          tab: currentTab,
+          interval: currentInterval,
+          domain: currentDomain,
+          window: currentWindow,
+          percent: modePercent ? 1 : 0,
+          group: modeGroup ? 1 : 0,
+          exclude_dash: excludeUncached ? 1 : 0,
+          smooth: smoothError ? 1 : 0,
+        }));
+      } catch {}
+    }
+
+    function loadSavedState() {
+      try {
+        const s = JSON.parse(localStorage.getItem(STATE_KEY) || '{}');
+        if (s.tab) currentTab = s.tab;
+        if (s.interval) currentInterval = s.interval;
+        if (s.domain !== undefined) currentDomain = s.domain;
+        if (s.window) { currentWindow = s.window; hadExplicitWindow = true; }
+        if (s.percent !== undefined) modePercent = !!Number(s.percent);
+        if (s.group !== undefined) modeGroup = !!Number(s.group);
+        if (s.exclude_dash !== undefined) excludeUncached = !!Number(s.exclude_dash);
+        if (s.smooth !== undefined) smoothError = !!Number(s.smooth);
+      } catch {}
+    }
+
+    function applyURLParams() {
+      const params = new URLSearchParams(location.search);
+      if (params.get('tab')) currentTab = params.get('tab');
+      if (params.get('interval')) currentInterval = params.get('interval');
+      if (params.get('domain') !== null) currentDomain = params.get('domain') || '';
+      if (params.get('window')) { currentWindow = params.get('window'); hadExplicitWindow = true; }
+      if (params.get('percent') !== null) modePercent = params.get('percent') === '1';
+      if (params.get('group') !== null) modeGroup = params.get('group') === '1';
+      if (params.get('exclude_dash') !== null) excludeUncached = params.get('exclude_dash') === '1';
+      if (params.get('smooth') !== null) smoothError = params.get('smooth') === '1';
+    }
+
+    function updateURL() {
+      const params = new URLSearchParams();
+      params.set('tab', currentTab);
+      params.set('interval', currentInterval);
+      if (currentDomain) params.set('domain', currentDomain);
+      params.set('window', currentWindow);
+      params.set('percent', modePercent ? '1' : '0');
+      params.set('group', modeGroup ? '1' : '0');
+      params.set('exclude_dash', excludeUncached ? '1' : '0');
+      params.set('smooth', smoothError ? '1' : '0');
+      const newUrl = `${location.pathname}?${params.toString()}`;
+      history.replaceState(null, '', newUrl);
+      saveState();
+    }
+
+    function bucketsForWindow(win, interval) {
+      switch (win) {
+        case '1h': return interval === 'hourly' ? 1 : 'all';
+        case '24h': return interval === 'hourly' ? 24 : 'all';
+        case '7d': return interval === 'daily' ? 7 : 'all';
+        case '30d': return interval === 'daily' ? 30 : 'all';
+        case '12w': return interval === 'weekly' ? 12 : 'all';
+        case '12m': return interval === 'monthly' ? 12 : 'all';
+        default: return 'all';
+      }
+    }
+
+    function availableIntervals() {
+      try {
+        return Array.from(intervalSelect ? intervalSelect.options : []).map(o => o.value);
+      } catch { return []; }
+    }
+
+    function pickIntervalForWindow(win) {
+      const avail = availableIntervals();
+      const pref = (list) => list.find(x => avail.includes(x));
+      switch (win) {
+        case '1h':
+        case '24h':
+          return pref(['hourly','daily','weekly','monthly']) || (avail[0] || 'daily');
+        case '7d':
+        case '30d':
+          return pref(['daily','weekly','monthly','hourly']) || (avail[0] || 'daily');
+        case '12w':
+          return pref(['weekly','daily','monthly']) || (avail[0] || 'weekly');
+        case '12m':
+          return pref(['monthly','weekly','daily']) || (avail[0] || 'monthly');
+        default:
+          // all time: favor coarser buckets if available
+          return pref(['monthly','weekly','daily','hourly']) || (avail[0] || 'weekly');
+      }
+    }
+
+    function applyTimePreset(win) {
+      currentWindow = win;
+      currentInterval = pickIntervalForWindow(win);
+      if (intervalSelect) intervalSelect.value = currentInterval;
+      const winSel = document.getElementById('window-select');
+      if (winSel) winSel.value = currentWindow;
+    }
+
+    function initReport(token, rep, base) {
+      fetch(base + '/' + rep.json, { signal: token.controller.signal })
         .then(r => r.json())
         .then(data => {
-          const bucketField = rep.bucket || 'bucket';
+          if (token !== currentLoad) return;
+          const bucketFields = rep.buckets || [rep.bucket || 'bucket'];
+          const labels = Array.isArray(rep.bucket_label)
+            ? rep.bucket_label
+            : [rep.bucket_label || 'Bucket'];
           if (rep.chart === 'table') {
-            const rows = data.map(x => [x[bucketField], x.value]);
-            new DataTable('#table-' + rep.name, {
+            const rows = data.map(x => bucketFields.map(f => x[f]).concat(x.value));
+            const columns = labels.map(l => ({ title: l }));
+            columns.push({ title: 'Value' });
+            const table = new DataTable('#table-' + rep.name, {
               data: rows,
-              columns: [
-                { title: rep.bucket_label || 'Bucket' },
-                { title: 'Value' }
-              ]
+              columns: columns
             });
+            registerChart(token, rep.name, table);
             return;
           }
 
-          const labels = data.map(x => x[bucketField]);
-          const values = data.map(x => x.value);
+          // Transform pipeline (client-only)
+          let transformed = data.slice();
+          const bucketField = bucketFields[0];
+          const isTimeSeries = bucketField === 'time_bucket';
+          // Exclusions (per-report) and explicit uncached toggle for cache_status
+          if (rep.exclude_values && rep.exclude_values.length) {
+            transformed = excludeValues(transformed, bucketField, rep.exclude_values);
+          }
+          if (excludeUncached && bucketField === 'cache_status') {
+            transformed = excludeValues(transformed, bucketField, ['-']);
+          }
+          // Windowing for time series
+          if (isTimeSeries) {
+            // Only apply windowing if report supports current window (if constrained)
+            const supported = Array.isArray(rep.windows_supported) ? rep.windows_supported : null;
+            const canWindow = !supported || supported.includes(currentWindow);
+            if (canWindow) {
+              const n = bucketsForWindow(currentWindow, currentInterval);
+              transformed = sliceWindow(transformed, n);
+            }
+          }
+          // Distributions: percent + group small
+          const isDistribution = ['pie', 'polarArea', 'doughnut', 'donut'].includes(rep.chart);
+          if (isDistribution) {
+            if (modeGroup) {
+              const thr = (typeof rep.group_others_threshold === 'number') ? rep.group_others_threshold : 0.03;
+              transformed = groupOthers(transformed, bucketField, 'value', thr, 'Other');
+            }
+            if (modePercent) {
+              transformed = toPercent(transformed, 'value');
+            }
+          }
+          // Relabel '-' to 'Uncached' for cache_status distributions
+          if (bucketField === 'cache_status') {
+            transformed = transformed.map(row => ({
+              ...row,
+              [bucketField]: row[bucketField] === '-' ? 'Uncached' : row[bucketField]
+            }));
+          }
+
+          const labelsArr = transformed.map(x => x[bucketField]);
+          let values = transformed.map(x => x.value);
           const chartType = rep.chart === 'stackedBar' ? 'bar' : rep.chart;
           const options = { scales: { y: { beginAtZero: true } } };
+          let datasets = [];
           if (rep.chart === 'stackedBar') {
             options.scales.x = { stacked: true };
+            options.scales.y = options.scales.y || {};
             options.scales.y.stacked = true;
-          }
+            // Build multiple series from columns (exclude bucket & total)
+            const keys = transformed.length ? Object.keys(transformed[0]).filter(k => k !== bucketField && k !== 'total') : [];
+            const palette = rep.colors || rep.palette || [
+              '#3273dc', '#23d160', '#ffdd57', '#ff3860', '#7957d5', '#363636'
+            ];
+            datasets = keys.map((k, i) => ({
+              label: k,
+              data: transformed.map(r => Number(r[k]) || 0),
+              backgroundColor: palette[i % palette.length],
+              borderColor: palette[i % palette.length],
+              borderWidth: 1,
+              fill: false,
+            }));
+          } else {
             const dataset = {
               label: rep.label,
               data: values,
@@ -139,6 +402,9 @@
             if (rep.colors) {
               dataset.backgroundColor = rep.colors;
               dataset.borderColor = rep.colors;
+            } else if (rep.palette) {
+              dataset.backgroundColor = rep.palette;
+              dataset.borderColor = rep.palette;
             } else if (rep.color) {
               dataset.backgroundColor = rep.color;
               dataset.borderColor = rep.color;
@@ -146,14 +412,22 @@
               dataset.backgroundColor = 'rgba(54, 162, 235, 0.5)';
               dataset.borderColor = 'rgba(54, 162, 235, 1)';
             }
-          new Chart(document.getElementById('chart-' + rep.name), {
+            // Optional smoothing for error_rate
+            if (rep.name === 'error_rate' && smoothError) {
+              dataset.data = movingAverage(values, 3);
+              dataset.label = rep.label + ' (smoothed)';
+            }
+            datasets = [dataset];
+          }
+          const chart = new Chart(document.getElementById('chart-' + rep.name), {
             type: chartType,
             data: {
-              labels: labels,
-              datasets: [dataset]
+              labels: labelsArr,
+              datasets
             },
             options: options
           });
+          registerChart(token, rep.name, chart);
         });
     }
 
@@ -171,49 +445,59 @@
         });
     }
 
-    function destroyCharts(container) {
-      container.querySelectorAll('canvas').forEach(c => {
-        const chart = Chart.getChart(c);
-        if (chart) {
-          chart.destroy();
-        }
-      });
-    }
-
-    function destroyAllCharts() {
-      Object.values(containers).forEach(destroyCharts);
-    }
+    // Reset helpers managed by chartManager
 
     function loadReports() {
       let path;
-      let container;
-      if (currentTab === 'overview') {
+      let container = containers[currentTab];
+      if (currentTab === 'recent') {
         path = 'global';
-        container = containers.overview;
-      } else if (currentTab === 'all') {
-        path = currentInterval;
-        container = containers.all;
       } else {
-        container = containers.domain;
-        if (!currentDomain) {
-          destroyCharts(container);
-          container.innerHTML = '<p>Select a domain</p>';
-          return;
-        }
-        path = 'domains/' + encodeURIComponent(currentDomain) + '/' + currentInterval;
+        path = currentDomain ? ('domains/' + encodeURIComponent(currentDomain) + '/' + currentInterval) : currentInterval;
       }
 
-        fetch(path + '/reports.json')
+      // Clear the top row on each load of Recent
+      if (currentTab === 'recent' && recentRow) {
+        recentRow.innerHTML = '';
+      }
+
+      const token = newLoad(container);
+
+      fetch(path + '/reports.json', { signal: token.controller.signal })
         .then(r => r.json())
         .then(reports => {
-            destroyCharts(container);
-            container.innerHTML = '';
-            reports.forEach(rep => {
-              fetch(path + '/' + rep.html)
+          if (token !== currentLoad) return;
+          const isDistributionType = t => ['pie','polarArea','doughnut','donut'].includes(t);
+          const filtered = reports.filter(rep => {
+            if (currentTab === 'recent') return true;
+            if (currentTab === 'trends') return rep.chart !== 'table' && !isDistributionType(rep.chart);
+            if (currentTab === 'breakdown') return isDistributionType(rep.chart) || rep.chart === 'table';
+            return true;
+          });
+          // If no explicit window was given (URL or saved state), honor first report's default
+          if (!hadExplicitWindow) {
+            const withDefault = filtered.find(r => r.window_default);
+            if (withDefault && typeof withDefault.window_default === 'string') {
+              currentWindow = withDefault.window_default;
+              windowSelect.value = currentWindow;
+              updateURL();
+            }
+          }
+          filtered.forEach(rep => {
+            fetch(path + '/' + rep.html, { signal: token.controller.signal })
               .then(r => r.text())
               .then(html => {
+                if (token !== currentLoad) return;
+                // On Recent tab, render Cache Status and HTTP Statuses side-by-side
+                const inTopRow = currentTab === 'recent' &&
+                  (rep.name === 'cache_status_breakdown' || rep.name === 'status_distribution');
+                if (inTopRow && recentRow) {
+                  const wrapped = `<div class="column is-half">${html}</div>`;
+                  recentRow.insertAdjacentHTML('beforeend', wrapped);
+                } else {
                   container.insertAdjacentHTML('beforeend', html);
-                  initReport(rep, path);
+                }
+                initReport(token, rep, path);
               });
           });
         });
@@ -300,7 +584,8 @@
     }
 
     function switchTab(name) {
-      destroyAllCharts();
+      abortLoad(currentLoad);
+      Object.values(containers).forEach(reset);
       currentTab = name;
       tabs.forEach(tab => {
         tab.classList.toggle('is-active', tab.dataset.tab === name);
@@ -308,9 +593,20 @@
       Object.entries(sections).forEach(([key, section]) => {
         section.classList.toggle('is-hidden', key !== name);
       });
-      intervalControl.classList.toggle('is-hidden', name === 'overview' || name === 'analysis');
-      domainControl.classList.toggle('is-hidden', name !== 'domain');
-      if (name === 'overview') {
+      const showTime = name !== 'recent' && name !== 'analysis';
+      const showDomain = showTime;
+      // Always keep legacy interval control hidden; use unified time control
+      intervalControl.classList.add('is-hidden');
+      domainControl.classList.toggle('is-hidden', !showDomain);
+      timeControl.classList.toggle('is-hidden', !showTime);
+      // Only show percent/group/exclude toggles on Breakdown tab,
+      // and smoothing only on Trends tab
+      modePercentControl.classList.toggle('is-hidden', name !== 'breakdown');
+      modeGroupControl.classList.toggle('is-hidden', name !== 'breakdown');
+      excludeUncachedControl.classList.toggle('is-hidden', name !== 'breakdown');
+      smoothControl.classList.toggle('is-hidden', name !== 'trends');
+      updateURL();
+      if (name === 'recent') {
         loadStats();
       }
       if (name === 'analysis') {
@@ -320,21 +616,103 @@
       }
     }
 
+    if (intervalSelect) {
       intervalSelect.addEventListener('change', () => {
         currentInterval = intervalSelect.value;
+        abortLoad(currentLoad);
+        Object.values(containers).forEach(reset);
+        updateURL();
         loadReports();
       });
+    }
 
     domainSelect.addEventListener('change', () => {
       currentDomain = domainSelect.value;
+      abortLoad(currentLoad);
+      Object.values(containers).forEach(reset);
+      updateURL();
+      loadReports();
+    });
+
+    if (timeSelect) {
+      timeSelect.addEventListener('change', () => {
+        applyTimePreset(timeSelect.value);
+        abortLoad(currentLoad);
+        updateURL();
+        loadReports();
+      });
+    }
+
+    percentToggle.addEventListener('change', () => {
+      modePercent = percentToggle.checked;
+      abortLoad(currentLoad);
+      updateURL();
+      loadReports();
+    });
+
+    groupToggle.addEventListener('change', () => {
+      modeGroup = groupToggle.checked;
+      abortLoad(currentLoad);
+      updateURL();
+      loadReports();
+    });
+
+    excludeUncachedToggle.addEventListener('change', () => {
+      excludeUncached = excludeUncachedToggle.checked;
+      abortLoad(currentLoad);
+      updateURL();
+      loadReports();
+    });
+
+    smoothToggle.addEventListener('change', () => {
+      smoothError = smoothToggle.checked;
+      abortLoad(currentLoad);
+      updateURL();
       loadReports();
     });
 
     tabs.forEach(tab => {
-      tab.addEventListener('click', () => switchTab(tab.dataset.tab));
+      tab.addEventListener('click', () => {
+        switchTab(tab.dataset.tab);
       });
-
-    switchTab('overview');
+    });
+    resetButton.addEventListener('click', () => {
+      try {
+        localStorage.removeItem('ngxstat-state'); // clear legacy
+        localStorage.removeItem(STATE_KEY);
+      } catch {}
+      // Reset to hard defaults
+      currentTab = 'recent';
+      currentInterval = intervalSelect ? (intervalSelect.value = intervalSelect.options[0]?.value || currentInterval) : currentInterval;
+      currentDomain = domainSelect.value = '';
+      applyTimePreset('7d');
+      if (timeSelect) timeSelect.value = '7d';
+      modePercent = percentToggle.checked = false;
+      modeGroup = groupToggle.checked = true;
+      excludeUncached = excludeUncachedToggle.checked = true;
+      smoothError = smoothToggle.checked = false;
+      hadExplicitWindow = false;
+      switchTab(currentTab);
+    });
+    // Initialize state (URL -> localStorage -> defaults)
+    loadSavedState();
+    applyURLParams();
+    // Sync controls
+    if (intervalSelect) intervalSelect.value = currentInterval;
+    domainSelect.value = currentDomain;
+    // Sync unified time select based on state
+    if (timeSelect) {
+      const known = new Set(['1h','24h','7d','30d','12w','12m','all']);
+      const pick = known.has(currentWindow) ? currentWindow : 'all';
+      timeSelect.value = pick;
+      applyTimePreset(pick);
+    }
+    percentToggle.checked = modePercent;
+    groupToggle.checked = modeGroup;
+    excludeUncachedToggle.checked = excludeUncached;
+    smoothToggle.checked = smoothError;
+    // Show/hide controls based on active tab
+    switchTab(currentTab);
   </script>
 </body>
 </html>

tests/test_analyze.py

@@ -1,12 +1,6 @@
-import sys
 import json
 import sqlite3
 from pathlib import Path
-
-import pytest
-
-REPO_ROOT = Path(__file__).resolve().parents[1]
-sys.path.append(str(REPO_ROOT))
 from scripts import analyze
 from scripts import generate_reports as gr
 

tests/test_nginx_config.py

@@ -1,9 +1,3 @@
-import sys
-from pathlib import Path
-import pytest
-
-REPO_ROOT = Path(__file__).resolve().parents[1]
-sys.path.append(str(REPO_ROOT))
 from scripts import nginx_config as nc
 
 

tests/test_reports.py

@@ -1,12 +1,10 @@
 import sqlite3
 from pathlib import Path
 import json
-import sys
+from datetime import datetime
 
 import pytest
-
-REPO_ROOT = Path(__file__).resolve().parents[1]
-sys.path.append(str(REPO_ROOT))
+from typer.testing import CliRunner
 from scripts import generate_reports as gr
 
 
@@ -199,9 +197,25 @@ def test_generate_root_index(tmp_path, sample_reports, monkeypatch):
     assert '<option value="Global">' not in content
     assert '<option value="analysis">' not in content
 
-    # check for domain options
-    assert '<option value="foo.com">' in content
-    assert '<option value="bar.com">' in content
+
+def test_generated_marker_written(tmp_path, monkeypatch):
+    out_dir = tmp_path / "output"
+    monkeypatch.setattr(gr, "OUTPUT_DIR", out_dir)
+    monkeypatch.setattr(
+        gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
+    )
+    monkeypatch.setattr(gr, "GENERATED_MARKER", out_dir / "generated.txt")
+    monkeypatch.setattr(gr, "_copy_icons", lambda: None)
+    (out_dir / "hourly").mkdir(parents=True)
+
+    runner = CliRunner()
+    result = runner.invoke(gr.app, ["index"])
+    assert result.exit_code == 0, result.output
+
+    marker = out_dir / "generated.txt"
+    assert marker.exists()
+    content = marker.read_text().strip()
+    datetime.strptime(content, "%Y-%m-%d %H:%M:%S")
 
 
 def test_global_reports_once(tmp_path, sample_reports, monkeypatch):
@@ -253,3 +267,149 @@ def test_global_stats_file(tmp_path, sample_reports, monkeypatch):
     assert stats["unique_domains"] == 1
     assert isinstance(stats["generated_at"], str)
     assert stats["generation_seconds"] >= 0
+
+
+def test_multi_bucket_table(tmp_path, monkeypatch):
+    db_path = tmp_path / "database" / "ngxstat.db"
+    setup_db(db_path)
+    # add a second domain entry
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+    cur.execute(
+        "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+        (
+            "127.0.0.1",
+            "foo.com",
+            "2024-01-01 10:10:00",
+            "GET /foo HTTP/1.1",
+            200,
+            100,
+            "-",
+            "curl",
+            "MISS",
+        ),
+    )
+    conn.commit()
+    conn.close()
+
+    cfg = tmp_path / "reports.yml"
+    cfg.write_text(
+        """
+- name: multi
+  chart: table
+  global: true
+  buckets: [domain, agent]
+  bucket_label: [Domain, Agent]
+  query: |
+    SELECT host AS domain, user_agent AS agent, COUNT(*) AS value
+    FROM logs
+    GROUP BY host, agent
+"""
+    )
+
+    monkeypatch.setattr(gr, "DB_PATH", db_path)
+    monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
+    monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
+    monkeypatch.setattr(
+        gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
+    )
+
+    gr._generate_global()
+    gr._generate_interval("hourly")
+
+    data = json.loads((tmp_path / "output" / "global" / "multi.json").read_text())
+    assert {"domain", "agent", "value"} <= data[0].keys()
+    reports = json.loads((tmp_path / "output" / "global" / "reports.json").read_text())
+    entry = next(r for r in reports if r["name"] == "multi")
+    assert entry["buckets"] == ["domain", "agent"]
+    assert entry["bucket_label"] == ["Domain", "Agent"]
+
+
+def test_top_n_limit_applied(tmp_path, monkeypatch):
+    # Prepare DB with many distinct agents
+    db_path = tmp_path / "database" / "ngxstat.db"
+    setup_db(db_path)
+    conn = sqlite3.connect(db_path)
+    cur = conn.cursor()
+    for i in range(10):
+        cur.execute(
+            "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (
+                "127.0.0.1",
+                "example.com",
+                f"2024-01-01 11:{i:02d}:00",
+                "GET /x HTTP/1.1",
+                200,
+                100,
+                "-",
+                f"ua-{i}",
+                "MISS",
+            ),
+        )
+    conn.commit()
+    conn.close()
+
+    cfg = tmp_path / "reports.yml"
+    cfg.write_text(
+        """
+- name: agents
+  chart: table
+  global: true
+  top_n: 3
+  query: |
+    SELECT user_agent AS agent, COUNT(*) AS value
+    FROM logs
+    GROUP BY user_agent
+    ORDER BY value DESC
+"""
+    )
+
+    monkeypatch.setattr(gr, "DB_PATH", db_path)
+    monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
+    monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
+    monkeypatch.setattr(
+        gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
+    )
+
+    gr._generate_global()
+
+    data = json.loads((tmp_path / "output" / "global" / "agents.json").read_text())
+    # Should be limited to 3 rows
+    assert len(data) <= 3
+
+
+def test_metadata_passthrough(tmp_path, monkeypatch):
+    db_path = tmp_path / "database" / "ngxstat.db"
+    setup_db(db_path)
+
+    cfg = tmp_path / "reports.yml"
+    cfg.write_text(
+        """
+- name: custom_ts
+  label: Custom TS
+  chart: line
+  window_default: 24h
+  windows_supported: [1h, 24h, 7d]
+  palette: ["#111111", "#222222"]
+  query: |
+    SELECT {bucket} AS time_bucket, COUNT(*) AS value
+    FROM logs
+    GROUP BY time_bucket
+    ORDER BY time_bucket
+"""
+    )
+
+    monkeypatch.setattr(gr, "DB_PATH", db_path)
+    monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
+    monkeypatch.setattr(gr, "REPORT_CONFIG", cfg)
+    monkeypatch.setattr(
+        gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
+    )
+
+    gr._generate_interval("hourly")
+
+    reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text())
+    entry = next(r for r in reports if r["name"] == "custom_ts")
+    assert entry["window_default"] == "24h"
+    assert entry["windows_supported"] == ["1h", "24h", "7d"]
+    assert entry["palette"] == ["#111111", "#222222"]

tests/test_run_analysis.py

@@ -16,7 +16,7 @@ def test_script_invokes_commands(tmp_path):
     python_stub = tmp_path / "python"
     python_stub.write_text(f'#!/usr/bin/env bash\necho "$*" >> "{calls}"\n')
     python_stub.chmod(0o755)
-    (tmp_path / "python3").write_text(f"#!/usr/bin/env bash\nexit 0\n")
+    (tmp_path / "python3").write_text("#!/usr/bin/env bash\nexit 0\n")
     (tmp_path / "python3").chmod(0o755)
 
     env = os.environ.copy()