Build Nginx live traffic monitor

app/__init__.py

@@ -0,0 +1 @@
+"""Nginx live traffic monitor."""

app/main.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import math
+import os
+import statistics
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, AsyncIterator
+
+from fastapi import FastAPI, Request
+from fastapi.responses import FileResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+
+
+DEFAULT_LOG = "/var/log/nginx-traffic/traffic.log"
+MAX_HOSTS = 1000
+
+
+@dataclass
+class Bucket:
+    incoming: int = 0
+    outgoing: int = 0
+    requests: int = 0
+    statuses: list[int] = field(default_factory=lambda: [0, 0, 0, 0])
+    request_times: list[float] = field(default_factory=list)
+    upstream_times: list[float] = field(default_factory=list)
+
+
+class Series:
+    def __init__(self, retention: int):
+        self.retention = retention
+        self.buckets: dict[int, Bucket] = {}
+
+    def bucket(self, second: int) -> Bucket:
+        if second not in self.buckets:
+            self.buckets[second] = Bucket()
+        cutoff = second - self.retention + 1
+        for old in [key for key in self.buckets if key < cutoff]:
+            del self.buckets[old]
+        return self.buckets[second]
+
+
+def percentile(values: list[float], p: float) -> float | None:
+    if not values:
+        return None
+    values = sorted(values)
+    index = (len(values) - 1) * p
+    low, high = math.floor(index), math.ceil(index)
+    if low == high:
+        return values[low]
+    return values[low] + (values[high] - values[low]) * (index - low)
+
+
+def parse_line(line: str) -> dict[str, Any] | None:
+    """Parse the configured, whitespace-delimited Nginx traffic format."""
+    fields = line.strip().split()
+    if len(fields) < 8:
+        return None
+    try:
+        timestamp = float(fields[0])
+        status = int(fields[3])
+        request_length = int(fields[4])
+        bytes_sent = int(fields[5])
+        request_time = float(fields[6])
+        upstream_raw = "".join(fields[7:])
+        upstream_values = [float(value) for value in upstream_raw.split(",") if value not in ("", "-")]
+        if not all(math.isfinite(value) and value >= 0 for value in
+                   (timestamp, request_length, bytes_sent, request_time, *upstream_values)):
+            return None
+        if status < 100 or status > 599:
+            return None
+    except (ValueError, OverflowError):
+        return None
+    return {
+        "timestamp": timestamp,
+        "host": fields[1] or "(unknown)",
+        "incoming": request_length,
+        "outgoing": bytes_sent,
+        "status": status,
+        "request_time": request_time,
+        "upstream_time": statistics.fmean(upstream_values) if upstream_values else None,
+    }
+
+
+class Telemetry:
+    def __init__(self, retention: int):
+        self.retention = max(60, min(retention, 86400))
+        self.global_series = Series(self.retention)
+        self.hosts: dict[str, Series] = {}
+        self.last_seen: dict[str, int] = {}
+        self.total_incoming = 0
+        self.total_outgoing = 0
+        self.lock = threading.RLock()
+        self.subscribers: set[asyncio.Queue[str]] = set()
+
+    def ingest(self, event: dict[str, Any]) -> None:
+        now = int(time.time())
+        second = int(event["timestamp"])
+        # Log timestamps are normally current, but clamping protects bounded
+        # storage if a clock-jumped or otherwise surprising line appears.
+        if second < now - self.retention:
+            return
+        second = min(second, now)
+        with self.lock:
+            self._add(self.global_series.bucket(second), event)
+            host = event["host"]
+            if host not in self.hosts:
+                if len(self.hosts) >= MAX_HOSTS:
+                    oldest = min(self.last_seen, key=self.last_seen.get)
+                    self.hosts.pop(oldest, None)
+                    self.last_seen.pop(oldest, None)
+                self.hosts[host] = Series(self.retention)
+            self._add(self.hosts[host].bucket(second), event)
+            self.last_seen[host] = second
+            self.total_incoming += event["incoming"]
+            self.total_outgoing += event["outgoing"]
+
+    @staticmethod
+    def _add(bucket: Bucket, event: dict[str, Any]) -> None:
+        bucket.incoming += event["incoming"]
+        bucket.outgoing += event["outgoing"]
+        bucket.requests += 1
+        status_class = event["status"] // 100
+        if 2 <= status_class <= 5:
+            bucket.statuses[status_class - 2] += 1
+        if len(bucket.request_times) < 256:
+            bucket.request_times.append(event["request_time"])
+        if event["upstream_time"] is not None and len(bucket.upstream_times) < 256:
+            bucket.upstream_times.append(event["upstream_time"])
+
+    def subscribe(self) -> asyncio.Queue[str]:
+        queue: asyncio.Queue[str] = asyncio.Queue(maxsize=2)
+        with self.lock:
+            self.subscribers.add(queue)
+        return queue
+
+    def unsubscribe(self, queue: asyncio.Queue[str]) -> None:
+        with self.lock:
+            self.subscribers.discard(queue)
+
+    def snapshot(self, window: int) -> dict[str, Any]:
+        now = int(time.time())
+        start = now - window + 1
+        with self.lock:
+            points = [self._point(self.global_series.buckets.get(second), second) for second in range(start, now + 1)]
+            hosts = sorted(self.hosts, key=lambda host: self.last_seen.get(host, 0), reverse=True)
+            active = [host for host in hosts if self.last_seen.get(host, 0) >= now - 59]
+            host_points = {
+                host: [self._point(self.hosts[host].buckets.get(second), second) for second in range(start, now + 1)]
+                for host in hosts
+            }
+            current = self._summary(points[-60:])
+            current["total_incoming"] = self.total_incoming
+            current["total_outgoing"] = self.total_outgoing
+            current["active_hosts"] = len(active)
+            current["busiest_host"] = max(active, key=lambda host: sum(p["incoming"] + p["outgoing"] for p in host_points[host][-60:]), default=None)
+            return {"type": "telemetry", "now": now, "points": points, "hosts": hosts, "host_points": host_points, "summary": current}
+
+    @staticmethod
+    def _point(bucket: Bucket | None, second: int) -> dict[str, Any]:
+        if not bucket:
+            return {"time": second, "incoming": 0, "outgoing": 0, "requests": 0, "status": [0, 0, 0, 0], "latency": None, "p95": None, "upstream": None}
+        return {"time": second, "incoming": bucket.incoming, "outgoing": bucket.outgoing, "requests": bucket.requests, "status": bucket.statuses, "latency": statistics.fmean(bucket.request_times) if bucket.request_times else None, "p95": percentile(bucket.request_times, .95), "upstream": statistics.fmean(bucket.upstream_times) if bucket.upstream_times else None}
+
+    @staticmethod
+    def _summary(points: list[dict[str, Any]]) -> dict[str, Any]:
+        seconds = max(1, len(points))
+        total_requests = sum(point["requests"] for point in points)
+        statuses = [sum(point["status"][index] for point in points) for index in range(4)]
+        latencies = [point["latency"] for point in points if point["latency"] is not None]
+        upstream = [point["upstream"] for point in points if point["upstream"] is not None]
+        return {"incoming": sum(point["incoming"] for point in points) / seconds, "outgoing": sum(point["outgoing"] for point in points) / seconds, "requests": total_requests / seconds, "status": statuses, "latency": statistics.fmean(latencies) if latencies else None, "p95": percentile(latencies, .95), "upstream": statistics.fmean(upstream) if upstream else None}
+
+
+async def follow_log(path: str, telemetry: Telemetry, stop: asyncio.Event) -> None:
+    handle = None
+    inode = None
+    pending = ""
+    while not stop.is_set():
+        try:
+            stat = await asyncio.to_thread(os.stat, path)
+            if handle is None:
+                handle = await asyncio.to_thread(open, path, "r", encoding="utf-8", errors="replace")
+                await asyncio.to_thread(handle.seek, 0, os.SEEK_END)
+                inode = (stat.st_dev, stat.st_ino)
+            elif inode != (stat.st_dev, stat.st_ino):
+                # Drain what is currently available from the renamed file before switching.
+                for line in handle:
+                    event = parse_line(line)
+                    if event:
+                        telemetry.ingest(event)
+                handle.close()
+                handle = await asyncio.to_thread(open, path, "r", encoding="utf-8", errors="replace")
+                inode = (stat.st_dev, stat.st_ino)
+                # Never join a partial line from the old inode to a new file.
+                pending = ""
+            data = await asyncio.to_thread(handle.read)
+            if data:
+                pending += data
+                lines = pending.split("\n")
+                pending = lines.pop()
+                for line in lines:
+                    event = parse_line(line)
+                    if event:
+                        telemetry.ingest(event)
+            await asyncio.sleep(.15)
+        except FileNotFoundError:
+            if handle:
+                data = handle.read()
+                if data:
+                    pending += data
+            await asyncio.sleep(.5)
+        except (OSError, ValueError):
+            if handle:
+                handle.close()
+            handle, inode = None, None
+            await asyncio.sleep(.5)
+    if handle:
+        handle.close()
+
+
+def create_app(log_path: str = DEFAULT_LOG, retention: int = 3600) -> FastAPI:
+    telemetry = Telemetry(retention)
+    stop = asyncio.Event()
+    app = FastAPI(title="Nginx Live Traffic Monitor")
+    app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static")
+
+    @app.on_event("startup")
+    async def startup() -> None:
+        app.state.task = asyncio.create_task(follow_log(log_path, telemetry, stop))
+        app.state.publisher = asyncio.create_task(publish())
+
+    @app.on_event("shutdown")
+    async def shutdown() -> None:
+        stop.set()
+        for task in (getattr(app.state, "task", None), getattr(app.state, "publisher", None)):
+            if task:
+                task.cancel()
+
+    async def publish() -> None:
+        while not stop.is_set():
+            payload = json.dumps(telemetry.snapshot(3600), separators=(",", ":"))
+            with telemetry.lock:
+                subscribers = list(telemetry.subscribers)
+            for queue in subscribers:
+                try:
+                    queue.put_nowait(payload)
+                except asyncio.QueueFull:
+                    try:
+                        queue.get_nowait()
+                    except asyncio.QueueEmpty:
+                        pass
+                    queue.put_nowait(payload)
+            await asyncio.sleep(1)
+
+    @app.get("/")
+    async def index() -> FileResponse:
+        return FileResponse(Path(__file__).parent / "static" / "index.html")
+
+    @app.get("/api/telemetry")
+    async def telemetry_api(window: int = 300) -> dict[str, Any]:
+        return telemetry.snapshot(max(60, min(window, 3600)))
+
+    @app.get("/api/events")
+    async def events(request: Request, window: int = 300) -> StreamingResponse:
+        queue = telemetry.subscribe()
+        initial = json.dumps(telemetry.snapshot(max(60, min(window, 3600))), separators=(",", ":"))
+        async def stream() -> AsyncIterator[str]:
+            try:
+                yield f"data: {initial}\n\n"
+                while True:
+                    if await request.is_disconnected():
+                        break
+                    try:
+                        payload = await asyncio.wait_for(queue.get(), timeout=15)
+                        yield f"data: {payload}\n\n"
+                    except asyncio.TimeoutError:
+                        yield ": keep-alive\n\n"
+            finally:
+                telemetry.unsubscribe(queue)
+        return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
+
+    app.state.telemetry = telemetry
+    return app
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Live Nginx traffic dashboard")
+    parser.add_argument("--log-path", default=os.getenv("TRAFFIC_LOG_PATH", DEFAULT_LOG))
+    parser.add_argument("--host", default=os.getenv("LISTEN_ADDRESS", "127.0.0.1"))
+    parser.add_argument("--port", type=int, default=int(os.getenv("LISTEN_PORT", "8080")))
+    parser.add_argument("--retention", type=int, default=int(os.getenv("TELEMETRY_RETENTION", "3600")))
+    args = parser.parse_args()
+    import uvicorn
+    uvicorn.run(create_app(args.log_path, args.retention), host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

app/static/index.html

@@ -0,0 +1,54 @@
+<!doctype html>
+<html lang="en" data-theme="system">
+<head>
+  <meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Nginx Live Traffic Monitor</title>
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@1.0.2/css/bulma.min.css">
+  <style>
+    :root { --chart-grid: #d9e1ea; }
+    [data-theme="dark"] { --chart-grid: #3b4654; }
+    body { background: var(--bulma-scheme-main-bis); min-height: 100vh; }
+    .navbar, .card { background: var(--bulma-scheme-main); }
+    .chart-wrap { height: 390px; position: relative; }
+    .metric-value { font-size: 1.35rem; font-weight: 700; white-space: nowrap; }
+    .metric-label { color: var(--bulma-text-weak); font-size: .75rem; text-transform: uppercase; letter-spacing: .05em; }
+    .host-list { max-height: 185px; overflow-y: auto; }
+    .legend-dot { width: .65rem; height: .65rem; display: inline-block; border-radius: 50%; margin-right: .4rem; }
+    @media (max-width: 768px) { .chart-wrap { height: 280px; } }
+  </style>
+</head>
+<body>
+<nav class="navbar is-shadowless" role="navigation"><div class="container">
+  <div class="navbar-brand"><div class="navbar-item"><strong>Nginx Live Traffic</strong></div></div>
+  <div class="navbar-end"><div class="navbar-item"><div class="select is-small"><select id="theme"><option value="system">System</option><option value="light">Light</option><option value="dark">Dark</option></select></div></div></div>
+</div></nav>
+<main class="section pt-5"><div class="container">
+  <div class="level mb-4"><div><h1 class="title is-4 mb-1">Traffic overview</h1><p class="subtitle is-6 mb-0" id="connection">Connecting to telemetry…</p></div>
+    <div class="field has-addons"><p class="control"><span class="select is-small"><select id="window"><option value="60">1 minute</option><option value="300" selected>5 minutes</option><option value="900">15 minutes</option><option value="3600">1 hour</option></select></span></p></div>
+  </div>
+  <div class="columns is-multiline mb-2" id="metrics"></div>
+  <div class="card mb-5"><div class="card-content"><div class="level mb-3"><div class="level-left"><h2 class="title is-5 mb-0">Throughput</h2></div><div class="level-right"><span class="tag is-light" id="updated">Waiting for data</span></div></div><div class="chart-wrap"><canvas id="traffic-chart"></canvas></div></div></div>
+  <div class="card"><div class="card-content"><div class="columns"><div class="column is-one-quarter"><h2 class="title is-5">Host breakouts</h2><p class="help mb-3">Select hosts to compare their recent traffic.</p><div id="hosts" class="host-list"></div></div><div class="column"><div class="chart-wrap"><canvas id="host-chart"></canvas></div></div></div></div></div>
+</div></main>
+<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.4/dist/chart.umd.min.js"></script>
+<script>
+const palette = ['#3273dc','#23d160','#ff3860','#ffdd57','#7957d5','#00d1b2','#f14668','#209cee'];
+let selected = new Set(), latest = null, source;
+const $ = id => document.getElementById(id);
+function bytesPerSecond(value) { const bits = value * 8; if (bits >= 1e9) return (bits/1e9).toFixed(1)+' Gbit/s'; if (bits >= 1e6) return (bits/1e6).toFixed(1)+' Mbit/s'; if (bits >= 1e3) return (bits/1e3).toFixed(1)+' Kbit/s'; return Math.round(bits)+' bit/s'; }
+function duration(v) { return v == null ? '—' : v.toFixed(3)+' s'; }
+function applyTheme(value) { const actual = value === 'system' ? (matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light') : value; document.documentElement.dataset.theme = actual; document.documentElement.style.colorScheme = actual; if (typeof trafficChart !== 'undefined') { trafficChart.options.scales.x.grid.color = getComputedStyle(document.documentElement).getPropertyValue('--chart-grid'); hostChart.options.scales.x.grid.color = trafficChart.options.scales.x.grid.color; trafficChart.update(); hostChart.update(); } }
+function metric(label, value) { return `<div class="column is-half-mobile is-one-quarter-tablet"><div class="card"><div class="card-content py-4"><div class="metric-label">${label}</div><div class="metric-value">${value}</div></div></div></div>`; }
+function renderMetrics(s) { const status = s.status.reduce((a,b)=>a+b,0); const errors = status ? ((s.status[2]+s.status[3])/status*100).toFixed(1)+'%' : '0.0%'; $('metrics').innerHTML = metric('Inbound', bytesPerSecond(s.incoming))+metric('Outbound', bytesPerSecond(s.outgoing))+metric('Requests', s.requests.toFixed(1)+'/sec')+metric('Transferred', formatBytes(s.total_incoming+s.total_outgoing))+metric('Avg latency', duration(s.latency))+metric('p95 latency', duration(s.p95))+metric('Upstream avg', duration(s.upstream))+metric('Errors (4xx/5xx)', errors)+metric('Active hosts', s.active_hosts)+metric('Busiest host', s.busiest_host || '—'); }
+function formatBytes(n) { const units=['B','KB','MB','GB','TB']; let i=0; while(n>=1000 && i<units.length-1){n/=1000;i++;} return (i? n.toFixed(1):Math.round(n))+' '+units[i]; }
+function makeChart(id, datasets) { return new Chart($(id), {type:'line', data:{labels:[],datasets}, options:{animation:false, maintainAspectRatio:false, interaction:{mode:'index',intersect:false}, scales:{x:{grid:{color:getComputedStyle(document.documentElement).getPropertyValue('--chart-grid')},ticks:{maxTicksLimit:8}},y:{beginAtZero:true,ticks:{callback:v=>bytesPerSecond(v)}}}, plugins:{legend:{display:true}}}}); }
+const trafficChart = makeChart('traffic-chart',[{label:'Inbound',data:[],borderColor:'#3273dc',backgroundColor:'#3273dc',tension:.25,pointRadius:0},{label:'Outbound',data:[],borderColor:'#23d160',backgroundColor:'#23d160',tension:.25,pointRadius:0}]);
+const hostChart = makeChart('host-chart',[]);
+function render(data) { latest=data; const windowSize=Number($('window').value); const points=data.points.slice(-windowSize); trafficChart.data.labels=points.map(p=>new Date(p.time*1000).toLocaleTimeString([], {hour:'2-digit',minute:'2-digit',second:'2-digit'})); trafficChart.data.datasets[0].data=points.map(p=>p.incoming); trafficChart.data.datasets[1].data=points.map(p=>p.outgoing); trafficChart.update(); hostChart.data.labels=trafficChart.data.labels; hostChart.data.datasets=[...selected].map((host,i)=>({label:host,data:(data.host_points[host]||[]).slice(-windowSize).map(p=>p.incoming+p.outgoing),borderColor:palette[i%palette.length],tension:.25,pointRadius:0})); hostChart.update(); renderMetrics(data.summary); $('updated').textContent='Updated '+new Date().toLocaleTimeString(); }
+function connect() { if(source) source.close(); source=new EventSource('/api/events?window='+$('window').value); source.onopen=()=>{$('connection').textContent='Live · updates every second';}; source.onmessage=e=>render(JSON.parse(e.data)); source.onerror=()=>{$('connection').textContent='Reconnecting…';}; }
+function renderHosts(hosts) { $('hosts').innerHTML=hosts.map((host,i)=>`<label class="checkbox is-block mb-2"><input type="checkbox" value="${escapeHtml(host)}" ${selected.has(host)?'checked':''}> <span class="legend-dot" style="background:${palette[i%palette.length]}"></span>${escapeHtml(host)}</label>`).join('') || '<p class="has-text-grey">No hosts observed yet.</p>'; document.querySelectorAll('#hosts input').forEach(input=>input.onchange=()=>{input.checked?selected.add(input.value):selected.delete(input.value); if(latest) render(latest);}); }
+function escapeHtml(s) { return s.replace(/[&<>'"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;',"'":'&#39;','"':'&quot;'}[c])); }
+const oldRender=render; render=data=>{oldRender(data);renderHosts(data.hosts);};
+$('theme').value=localStorage.getItem('nginx-theme')||'system'; applyTheme($('theme').value); $('theme').onchange=()=>{localStorage.setItem('nginx-theme',$('theme').value);applyTheme($('theme').value);}; $('window').onchange=connect; connect();
+</script>
+</body></html>