wagesj45/ngxstat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #15 from wagesj45/codex/fix-operational-error-in-report-generation

Browse source 
Fix view generation for domain reports
This commit is contained in:
Jordan Wages 2025-07-18 02:16:25 -05:00 • committed by GitHub
commit 77e7cae1cf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 68 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

22
scripts/generate_reports.py
View file

@ -36,6 +36,7 @@ def _get_domains() -> List[str]:
conn.close()
return domains
def _load_config() -> List[Dict]:
if not REPORT_CONFIG.exists():
typer.echo(f"Config file not found: {REPORT_CONFIG}")
@ -47,10 +48,12 @@ def _load_config() -> List[Dict]:
raise typer.Exit(1)
return data
def _save_json(path: Path, data: List[Dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2))
def _render_html(interval: str, reports: List[Dict], out_path: Path) -> None:
env = Environment(loader=FileSystemLoader(TEMPLATE_DIR))
template = env.get_template("report.html")
@ -65,6 +68,7 @@ def _bucket_expr(interval: str) -> str:
raise typer.Exit(1)
return f"strftime('{fmt}', datetime(time))"
def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
cfg = _load_config()
if not cfg:
@ -79,9 +83,12 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
# Create a temporary view so queries can easily be filtered by domain
cur.execute("DROP VIEW IF EXISTS logs_view")
if domain:
# Parameters are not allowed in CREATE VIEW statements, so we must
# safely interpolate the domain value ourselves. Escape any single
# quotes to prevent malformed queries.
safe_domain = domain.replace("'", "''")
cur.execute(
"CREATE TEMP VIEW logs_view AS SELECT * FROM logs WHERE host = ?",
(domain,),
f"CREATE TEMP VIEW logs_view AS SELECT * FROM logs WHERE host = '{safe_domain}'"
)
out_dir = OUTPUT_DIR / domain / interval
else:
@ -101,12 +108,14 @@ def _generate_interval(interval: str, domain: Optional[str] = None) -> None:
data = [dict(zip(headers, row)) for row in rows]
json_path = out_dir / f"{name}.json"
_save_json(json_path, data)
report_list.append({
report_list.append(
{
"name": name,
"label": definition.get("label", name.title()),
"chart": definition.get("chart", "line"),
"json": f"{name}.json",
})
}
)
_save_json(out_dir / "reports.json", report_list)
_render_html(interval, report_list, out_dir / "index.html")
@ -118,6 +127,7 @@ def _generate_all_domains(interval: str) -> None:
for domain in _get_domains():
_generate_interval(interval, domain)
@app.command()
def hourly(
domain: Optional[str] = typer.Option(
@ -133,6 +143,7 @@ def hourly(
else:
_generate_interval("hourly", domain)
@app.command()
def daily(
domain: Optional[str] = typer.Option(
@ -148,6 +159,7 @@ def daily(
else:
_generate_interval("daily", domain)
@app.command()
def weekly(
domain: Optional[str] = typer.Option(
@ -163,6 +175,7 @@ def weekly(
else:
_generate_interval("weekly", domain)
@app.command()
def monthly(
domain: Optional[str] = typer.Option(
@ -178,5 +191,6 @@ def monthly(
else:
_generate_interval("monthly", domain)
if __name__ == "__main__":
app()

50
tests/test_reports.py
View file

@ -32,11 +32,31 @@ def setup_db(path: Path):
)
cur.execute(
"INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
("127.0.0.1", "example.com", "2024-01-01 10:00:00", "GET / HTTP/1.1", 200, 100, "-", "curl", "MISS"),
(
"127.0.0.1",
"example.com",
"2024-01-01 10:00:00",
"GET / HTTP/1.1",
200,
100,
"-",
"curl",
"MISS",
),
)
cur.execute(
"INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
("127.0.0.1", "example.com", "2024-01-01 10:05:00", "GET /err HTTP/1.1", 500, 100, "-", "curl", "MISS"),
(
"127.0.0.1",
"example.com",
"2024-01-01 10:05:00",
"GET /err HTTP/1.1",
500,
100,
"-",
"curl",
"MISS",
),
)
conn.commit()
conn.close()
@ -72,14 +92,36 @@ def test_generate_interval(tmp_path, sample_reports, monkeypatch):
monkeypatch.setattr(gr, "DB_PATH", db_path)
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
monkeypatch.setattr(gr, "REPORT_CONFIG", sample_reports)
monkeypatch.setattr(gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates")
monkeypatch.setattr(
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
)
gr._generate_interval("hourly")
hits = json.loads((tmp_path / "output" / "hourly" / "hits.json").read_text())
assert hits[0]["value"] == 2
error_rate = json.loads((tmp_path / "output" / "hourly" / "error_rate.json").read_text())
error_rate = json.loads(
(tmp_path / "output" / "hourly" / "error_rate.json").read_text()
)
assert error_rate[0]["value"] == pytest.approx(50.0)
reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text())
assert {r["name"] for r in reports} == {"hits", "error_rate"}
def test_generate_interval_domain_filter(tmp_path, sample_reports, monkeypatch):
db_path = tmp_path / "database" / "ngxstat.db"
setup_db(db_path)
monkeypatch.setattr(gr, "DB_PATH", db_path)
monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output")
monkeypatch.setattr(gr, "REPORT_CONFIG", sample_reports)
monkeypatch.setattr(
gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates"
)
gr._generate_interval("hourly", "example.com")
hits = json.loads(
(tmp_path / "output" / "example.com" / "hourly" / "hits.json").read_text()
)
assert hits[0]["value"] == 2