From d216433a0721145ea4685e6987214bc72c0f655b Mon Sep 17 00:00:00 2001 From: Jordan Wages Date: Thu, 17 Jul 2025 00:17:46 -0500 Subject: [PATCH] Update requirements and AGENTS instructions --- .gitignore | 3 ++ AGENTS.md | 6 ++- README.md | 23 ++++++++++- requirements.txt | 4 +- scripts/generate_reports.py | 79 +++++++++++++++++++++++++++++++++++++ templates/report.html | 42 ++++++++++++++++++++ 6 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 scripts/generate_reports.py create mode 100644 templates/report.html diff --git a/.gitignore b/.gitignore index a04f85b..e0316ca 100644 --- a/.gitignore +++ b/.gitignore @@ -211,3 +211,6 @@ database/*.db database/*.sqlite database/*.sqlite3 + +# Generated reports +output/ diff --git a/AGENTS.md b/AGENTS.md index 56bb510..ab65e99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,10 @@ This document outlines general practices and expectations for AI agents assistin ```bash python -m venv .venv source .venv/bin/activate -```` + pip install -r requirements.txt + ``` + The `init.sh` script can create this environment automatically. Always + activate it before running scripts or tests. * Dependency management: Use `requirements.txt` or `pip-tools` * Use standard libraries where feasible (e.g., `sqlite3`, `argparse`, `datetime`) @@ -96,3 +99,4 @@ As the project matures, agents may also: ## Changelog * **2025-07-17**: Initial version by Jordan + ChatGPT +* **2025-07-17**: Expanded virtual environment usage guidance diff --git a/README.md b/README.md index 3b69e39..bc2db2d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,25 @@ # ngxstat Per-domain Nginx log analytics with hybrid static reports and live insights. -## +## Generating Reports + +Use the `generate_reports.py` script to build aggregated JSON and HTML files from `database/ngxstat.db`. + +Create a virtual environment and install dependencies: + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +Then run one or more of the interval commands: + +```bash +python scripts/generate_reports.py hourly +python scripts/generate_reports.py daily +python scripts/generate_reports.py weekly +python scripts/generate_reports.py monthly +``` + +Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and produces an HTML dashboard using Chart.js. diff --git a/requirements.txt b/requirements.txt index a61cf87..221e3c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Core tools -typer[all] # For CLI commands -Jinja2 # For static HTML generation +typer[all]>=0.9 # For CLI commands +Jinja2>=3.1 # For static HTML generation sqlite-utils # Optional: high-level SQLite handling Flask # For optional lightweight API server diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py new file mode 100644 index 0000000..b244075 --- /dev/null +++ b/scripts/generate_reports.py @@ -0,0 +1,79 @@ +import json +import sqlite3 +from pathlib import Path +from typing import List, Dict + +import typer +from jinja2 import Environment, FileSystemLoader + +DB_PATH = Path("database/ngxstat.db") +OUTPUT_DIR = Path("output") +TEMPLATE_DIR = Path("templates") + +app = typer.Typer(help="Generate aggregated log reports") + +def _load_existing(path: Path) -> List[Dict]: + if path.exists(): + try: + return json.loads(path.read_text()) + except Exception: + return [] + return [] + +def _save_json(path: Path, data: List[Dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2)) + +def _render_html(interval: str, json_name: str, out_path: Path) -> None: + env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) + template = env.get_template("report.html") + out_path.write_text(template.render(interval=interval, json_path=json_name)) + +def _aggregate(interval: str, fmt: str) -> None: + json_path = OUTPUT_DIR / f"{interval}.json" + html_path = OUTPUT_DIR / f"{interval}.html" + + existing = _load_existing(json_path) + last_bucket = existing[-1]["bucket"] if existing else None + + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + query = f"SELECT strftime('{fmt}', datetime(time)) as bucket, COUNT(*) as hits FROM logs" + params = [] + if last_bucket: + query += " WHERE datetime(time) > datetime(?)" + params.append(last_bucket) + query += " GROUP BY bucket ORDER BY bucket" + + rows = cur.execute(query, params).fetchall() + for bucket, hits in rows: + existing.append({"bucket": bucket, "hits": hits}) + + existing.sort(key=lambda x: x["bucket"]) + _save_json(json_path, existing) + _render_html(interval, json_path.name, html_path) + typer.echo(f"Generated {json_path} and {html_path}") + +@app.command() +def hourly() -> None: + """Aggregate logs into hourly buckets.""" + _aggregate("hourly", "%Y-%m-%d %H:00:00") + +@app.command() +def daily() -> None: + """Aggregate logs into daily buckets.""" + _aggregate("daily", "%Y-%m-%d") + +@app.command() +def weekly() -> None: + """Aggregate logs into weekly buckets.""" + _aggregate("weekly", "%Y-%W") + +@app.command() +def monthly() -> None: + """Aggregate logs into monthly buckets.""" + _aggregate("monthly", "%Y-%m") + +if __name__ == "__main__": + app() diff --git a/templates/report.html b/templates/report.html new file mode 100644 index 0000000..e6dfee0 --- /dev/null +++ b/templates/report.html @@ -0,0 +1,42 @@ + + + + + {{ interval.title() }} Report + + + + +
+

{{ interval.title() }} Report

+ +
+ + +