Refactor reports to share definitions across intervals
This commit is contained in:
parent
99a6b45d4c
commit
657c30b260
4 changed files with 38 additions and 18 deletions
13
README.md
13
README.md
|
@ -27,18 +27,21 @@ Reports are written under the `output/` directory. Each command updates the corr
|
||||||
### Configuring Reports
|
### Configuring Reports
|
||||||
|
|
||||||
Report queries are defined in `reports.yml`. Each entry specifies the `name`,
|
Report queries are defined in `reports.yml`. Each entry specifies the `name`,
|
||||||
`interval`, optional `label` and `chart` type, and a SQL `query` that must return
|
optional `label` and `chart` type, and a SQL `query` that must return `bucket`
|
||||||
`bucket` and `value` columns. When `generate_reports.py` runs, every matching
|
and `value` columns. The special token `{bucket}` is replaced with the
|
||||||
definition creates `output/<interval>/<name>.json` and an interval dashboard.
|
appropriate SQLite `strftime` expression for each interval (hourly, daily,
|
||||||
|
weekly or monthly) so that a single definition works across all durations.
|
||||||
|
When `generate_reports.py` runs, every definition is executed for the requested
|
||||||
|
interval and creates `output/<interval>/<name>.json` along with an HTML
|
||||||
|
dashboard.
|
||||||
|
|
||||||
Example snippet:
|
Example snippet:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
- name: hits
|
- name: hits
|
||||||
interval: hourly
|
|
||||||
chart: bar
|
chart: bar
|
||||||
query: |
|
query: |
|
||||||
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket,
|
SELECT {bucket} AS bucket,
|
||||||
COUNT(*) AS value
|
COUNT(*) AS value
|
||||||
FROM logs
|
FROM logs
|
||||||
GROUP BY bucket
|
GROUP BY bucket
|
||||||
|
|
|
@ -1,20 +1,18 @@
|
||||||
- name: hits
|
- name: hits
|
||||||
interval: hourly
|
|
||||||
label: Hits
|
label: Hits
|
||||||
chart: bar
|
chart: bar
|
||||||
query: |
|
query: |
|
||||||
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket,
|
SELECT {bucket} AS bucket,
|
||||||
COUNT(*) AS value
|
COUNT(*) AS value
|
||||||
FROM logs
|
FROM logs
|
||||||
GROUP BY bucket
|
GROUP BY bucket
|
||||||
ORDER BY bucket
|
ORDER BY bucket
|
||||||
|
|
||||||
- name: error_rate
|
- name: error_rate
|
||||||
interval: hourly
|
|
||||||
label: Error Rate (%)
|
label: Error Rate (%)
|
||||||
chart: line
|
chart: line
|
||||||
query: |
|
query: |
|
||||||
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket,
|
SELECT {bucket} AS bucket,
|
||||||
SUM(CASE WHEN status >= 500 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value
|
SUM(CASE WHEN status >= 500 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value
|
||||||
FROM logs
|
FROM logs
|
||||||
GROUP BY bucket
|
GROUP BY bucket
|
||||||
|
|
|
@ -13,6 +13,17 @@ OUTPUT_DIR = Path("output")
|
||||||
TEMPLATE_DIR = Path("templates")
|
TEMPLATE_DIR = Path("templates")
|
||||||
REPORT_CONFIG = Path("reports.yml")
|
REPORT_CONFIG = Path("reports.yml")
|
||||||
|
|
||||||
|
# Mapping of interval names to SQLite strftime formats. These strings are
|
||||||
|
# substituted into report queries whenever the special ``{bucket}`` token is
|
||||||
|
# present so that a single report definition can be reused for multiple
|
||||||
|
# intervals.
|
||||||
|
INTERVAL_FORMATS = {
|
||||||
|
"hourly": "%Y-%m-%d %H:00:00",
|
||||||
|
"daily": "%Y-%m-%d",
|
||||||
|
"weekly": "%Y-%W",
|
||||||
|
"monthly": "%Y-%m",
|
||||||
|
}
|
||||||
|
|
||||||
app = typer.Typer(help="Generate aggregated log reports")
|
app = typer.Typer(help="Generate aggregated log reports")
|
||||||
|
|
||||||
def _load_config() -> List[Dict]:
|
def _load_config() -> List[Dict]:
|
||||||
|
@ -35,13 +46,23 @@ def _render_html(interval: str, reports: List[Dict], out_path: Path) -> None:
|
||||||
template = env.get_template("report.html")
|
template = env.get_template("report.html")
|
||||||
out_path.write_text(template.render(interval=interval, reports=reports))
|
out_path.write_text(template.render(interval=interval, reports=reports))
|
||||||
|
|
||||||
|
|
||||||
|
def _bucket_expr(interval: str) -> str:
|
||||||
|
"""Return the SQLite strftime expression for the given interval."""
|
||||||
|
fmt = INTERVAL_FORMATS.get(interval)
|
||||||
|
if not fmt:
|
||||||
|
typer.echo(f"Unsupported interval: {interval}")
|
||||||
|
raise typer.Exit(1)
|
||||||
|
return f"strftime('{fmt}', datetime(time))"
|
||||||
|
|
||||||
def _generate_interval(interval: str) -> None:
|
def _generate_interval(interval: str) -> None:
|
||||||
cfg = _load_config()
|
cfg = _load_config()
|
||||||
defs = [d for d in cfg if d.get("interval") == interval]
|
if not cfg:
|
||||||
if not defs:
|
typer.echo("No report definitions found")
|
||||||
typer.echo(f"No reports defined for {interval}")
|
|
||||||
return
|
return
|
||||||
|
|
||||||
|
bucket = _bucket_expr(interval)
|
||||||
|
|
||||||
conn = sqlite3.connect(DB_PATH)
|
conn = sqlite3.connect(DB_PATH)
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
@ -49,9 +70,9 @@ def _generate_interval(interval: str) -> None:
|
||||||
out_dir.mkdir(parents=True, exist_ok=True)
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
report_list = []
|
report_list = []
|
||||||
for definition in defs:
|
for definition in cfg:
|
||||||
name = definition["name"]
|
name = definition["name"]
|
||||||
query = definition["query"]
|
query = definition["query"].replace("{bucket}", bucket)
|
||||||
cur.execute(query)
|
cur.execute(query)
|
||||||
rows = cur.fetchall()
|
rows = cur.fetchall()
|
||||||
headers = [c[0] for c in cur.description]
|
headers = [c[0] for c in cur.description]
|
||||||
|
|
|
@ -48,16 +48,14 @@ def sample_reports(tmp_path):
|
||||||
cfg.write_text(
|
cfg.write_text(
|
||||||
"""
|
"""
|
||||||
- name: hits
|
- name: hits
|
||||||
interval: hourly
|
|
||||||
query: |
|
query: |
|
||||||
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket, COUNT(*) AS value
|
SELECT {bucket} AS bucket, COUNT(*) AS value
|
||||||
FROM logs
|
FROM logs
|
||||||
GROUP BY bucket
|
GROUP BY bucket
|
||||||
ORDER BY bucket
|
ORDER BY bucket
|
||||||
- name: error_rate
|
- name: error_rate
|
||||||
interval: hourly
|
|
||||||
query: |
|
query: |
|
||||||
SELECT strftime('%Y-%m-%d %H:00:00', datetime(time)) AS bucket,
|
SELECT {bucket} AS bucket,
|
||||||
SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value
|
SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value
|
||||||
FROM logs
|
FROM logs
|
||||||
GROUP BY bucket
|
GROUP BY bucket
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue