Refactor reports to share definitions across intervals

This commit is contained in:
Jordan Wages 2025-07-18 01:50:19 -05:00
commit 657c30b260
4 changed files with 38 additions and 18 deletions

View file

@ -27,18 +27,21 @@ Reports are written under the `output/` directory. Each command updates the corr
### Configuring Reports ### Configuring Reports
Report queries are defined in `reports.yml`. Each entry specifies the `name`, Report queries are defined in `reports.yml`. Each entry specifies the `name`,
`interval`, optional `label` and `chart` type, and a SQL `query` that must return optional `label` and `chart` type, and a SQL `query` that must return `bucket`
`bucket` and `value` columns. When `generate_reports.py` runs, every matching and `value` columns. The special token `{bucket}` is replaced with the
definition creates `output/<interval>/<name>.json` and an interval dashboard. appropriate SQLite `strftime` expression for each interval (hourly, daily,
weekly or monthly) so that a single definition works across all durations.
When `generate_reports.py` runs, every definition is executed for the requested
interval and creates `output/<interval>/<name>.json` along with an HTML
dashboard.
Example snippet: Example snippet:
```yaml ```yaml
- name: hits - name: hits
interval: hourly
chart: bar chart: bar
query: | query: |
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket, SELECT {bucket} AS bucket,
COUNT(*) AS value COUNT(*) AS value
FROM logs FROM logs
GROUP BY bucket GROUP BY bucket

View file

@ -1,20 +1,18 @@
- name: hits - name: hits
interval: hourly
label: Hits label: Hits
chart: bar chart: bar
query: | query: |
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket, SELECT {bucket} AS bucket,
COUNT(*) AS value COUNT(*) AS value
FROM logs FROM logs
GROUP BY bucket GROUP BY bucket
ORDER BY bucket ORDER BY bucket
- name: error_rate - name: error_rate
interval: hourly
label: Error Rate (%) label: Error Rate (%)
chart: line chart: line
query: | query: |
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket, SELECT {bucket} AS bucket,
SUM(CASE WHEN status >= 500 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value SUM(CASE WHEN status >= 500 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value
FROM logs FROM logs
GROUP BY bucket GROUP BY bucket

View file

@ -13,6 +13,17 @@ OUTPUT_DIR = Path("output")
TEMPLATE_DIR = Path("templates") TEMPLATE_DIR = Path("templates")
REPORT_CONFIG = Path("reports.yml") REPORT_CONFIG = Path("reports.yml")
# Mapping of interval names to SQLite strftime formats. These strings are
# substituted into report queries whenever the special ``{bucket}`` token is
# present so that a single report definition can be reused for multiple
# intervals.
INTERVAL_FORMATS = {
"hourly": "%Y-%m-%d %H:00:00",
"daily": "%Y-%m-%d",
"weekly": "%Y-%W",
"monthly": "%Y-%m",
}
app = typer.Typer(help="Generate aggregated log reports") app = typer.Typer(help="Generate aggregated log reports")
def _load_config() -> List[Dict]: def _load_config() -> List[Dict]:
@ -35,13 +46,23 @@ def _render_html(interval: str, reports: List[Dict], out_path: Path) -> None:
template = env.get_template("report.html") template = env.get_template("report.html")
out_path.write_text(template.render(interval=interval, reports=reports)) out_path.write_text(template.render(interval=interval, reports=reports))
def _bucket_expr(interval: str) -> str:
"""Return the SQLite strftime expression for the given interval."""
fmt = INTERVAL_FORMATS.get(interval)
if not fmt:
typer.echo(f"Unsupported interval: {interval}")
raise typer.Exit(1)
return f"strftime('{fmt}', datetime(time))"
def _generate_interval(interval: str) -> None: def _generate_interval(interval: str) -> None:
cfg = _load_config() cfg = _load_config()
defs = [d for d in cfg if d.get("interval") == interval] if not cfg:
if not defs: typer.echo("No report definitions found")
typer.echo(f"No reports defined for {interval}")
return return
bucket = _bucket_expr(interval)
conn = sqlite3.connect(DB_PATH) conn = sqlite3.connect(DB_PATH)
cur = conn.cursor() cur = conn.cursor()
@ -49,9 +70,9 @@ def _generate_interval(interval: str) -> None:
out_dir.mkdir(parents=True, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
report_list = [] report_list = []
for definition in defs: for definition in cfg:
name = definition["name"] name = definition["name"]
query = definition["query"] query = definition["query"].replace("{bucket}", bucket)
cur.execute(query) cur.execute(query)
rows = cur.fetchall() rows = cur.fetchall()
headers = [c[0] for c in cur.description] headers = [c[0] for c in cur.description]

View file

@ -48,16 +48,14 @@ def sample_reports(tmp_path):
cfg.write_text( cfg.write_text(
""" """
- name: hits - name: hits
interval: hourly
query: | query: |
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket, COUNT(*) AS value SELECT {bucket} AS bucket, COUNT(*) AS value
FROM logs FROM logs
GROUP BY bucket GROUP BY bucket
ORDER BY bucket ORDER BY bucket
- name: error_rate - name: error_rate
interval: hourly
query: | query: |
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket, SELECT {bucket} AS bucket,
SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value
FROM logs FROM logs
GROUP BY bucket GROUP BY bucket