diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..95f9808 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +exclude = .git, .venv, output, static/icons +max-line-length = 160 diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..5cf26be --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,151 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + ci: + name: Lint, test, and build + # This label must match your Forgejo runner's label + runs-on: docker + # Use a clean Debian container so tools are predictable + container: debian:stable-slim + env: + PYTHONDONTWRITEBYTECODE: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" + UV_SYSTEM_PYTHON: "1" + steps: + - name: Install build tooling + run: | + set -euo pipefail + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git ca-certificates python3 python3-venv python3-pip python3-setuptools \ + python3-wheel sqlite3 + update-ca-certificates || true + + - name: Checkout repository (manual) + run: | + set -euo pipefail + if [ -f Makefile ] || [ -d .git ]; then + echo "Repository present in workspace; skipping clone" + exit 0 + fi + REMOTE_URL="${CI_REPOSITORY_URL:-}" + if [ -z "$REMOTE_URL" ]; then + if [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then + REMOTE_URL="${GITHUB_SERVER_URL%/}/${GITHUB_REPOSITORY}.git" + elif [ -n "${GITHUB_REPOSITORY:-}" ]; then + REMOTE_URL="https://git.jordanwages.com/${GITHUB_REPOSITORY}.git" + else + echo "Unable to determine repository URL from CI environment" >&2 + exit 1 + fi + fi + AUTH_URL="$REMOTE_URL" + if [ -n "${GITHUB_TOKEN:-}" ]; then + ACTOR="${GITHUB_ACTOR:-oauth2}" + AUTH_URL=$(printf '%s' "$REMOTE_URL" | sed -E "s#^https://#https://${ACTOR}:${GITHUB_TOKEN}@#") + fi + echo "Cloning from: $REMOTE_URL" + if ! git clone --depth 1 "$AUTH_URL" .; then + echo "Auth clone failed; trying anonymous clone..." >&2 + git clone --depth 1 "$REMOTE_URL" . + fi + if [ -n "${GITHUB_SHA:-}" ]; then + git fetch --depth 1 origin "$GITHUB_SHA" || true + git checkout -q "$GITHUB_SHA" || true + elif [ -n "${GITHUB_REF_NAME:-}" ]; then + git fetch --depth 1 origin "$GITHUB_REF_NAME" || true + git checkout -q "$GITHUB_REF_NAME" || true + fi + + - name: Set up venv and install deps + run: | + set -euo pipefail + # Prefer persistent cache if runner provides /cache + USE_CACHE=0 + if [ -d /cache ] && [ -w /cache ]; then + export PIP_CACHE_DIR=/cache/pip + mkdir -p "$PIP_CACHE_DIR" + REQ_HASH=$(sha256sum requirements.txt | awk '{print $1}') + PYVER=$(python3 -c 'import sys;print(".".join(map(str, sys.version_info[:2])))') + CACHE_VENV="/cache/venv-${REQ_HASH}-py${PYVER}" + if [ ! -f "$CACHE_VENV/bin/activate" ]; then + echo "Preparing cached virtualenv: $CACHE_VENV" + rm -rf "$CACHE_VENV" || true + python3 -m venv "$CACHE_VENV" + fi + ln -sfn "$CACHE_VENV" .venv + USE_CACHE=1 + else + # Fallback to local venv + python3 -m venv .venv + fi + + # If the link didn't produce an activate file, fallback to local venv + if [ ! -f .venv/bin/activate ]; then + echo "Cached venv missing; creating local .venv" + rm -f .venv + python3 -m venv .venv + USE_CACHE=0 + fi + + . .venv/bin/activate + python -m pip install --upgrade pip + if [ "$USE_CACHE" = "1" ]; then + # Ensure required packages are present; pip will use cache + pip install -r requirements.txt pytest || pip install -r requirements.txt pytest + else + pip install -r requirements.txt pytest + fi + + - name: Format check (black) + run: | + . .venv/bin/activate + black --check . + + - name: Lint (flake8) + run: | + . .venv/bin/activate + flake8 . + + - name: Run tests (pytest) + run: | + . .venv/bin/activate + export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}" + pytest -q --maxfail=1 + + - name: Build sample reports (no artifact upload) + run: | + set -euo pipefail + . .venv/bin/activate + python - <<'PY' + import sqlite3, pathlib + db = pathlib.Path('database/ngxstat.db') + db.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db) + cur = conn.cursor() + cur.execute('''CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY, + ip TEXT, + host TEXT, + time TEXT, + request TEXT, + status INTEGER, + bytes_sent INTEGER, + referer TEXT, + user_agent TEXT, + cache_status TEXT + )''') + cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:00:00','GET / HTTP/1.1',200,100,'-','curl','MISS')") + cur.execute("INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES ('127.0.0.1','example.com','2024-01-01 10:05:00','GET /about HTTP/1.1',200,100,'-','curl','MISS')") + conn.commit(); conn.close() + PY + python scripts/generate_reports.py global + python scripts/generate_reports.py hourly + python scripts/generate_reports.py index + tar -czf ngxstat-reports.tar.gz -C output . + echo "Built sample reports archive: ngxstat-reports.tar.gz" diff --git a/AGENTS.md b/AGENTS.md index 50a1938..7e7d3c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,9 @@ This document outlines general practices and expectations for AI agents assistin The `run-import.sh` script can initialize this environment automatically. Always activate the virtual environment before running scripts or tests. +* Before committing code run `black` for consistent formatting and execute + the test suite with `pytest`. All tests should pass. + * Dependency management: Use `requirements.txt` or `pip-tools` * Use standard libraries where feasible (e.g., `sqlite3`, `argparse`, `datetime`) * Adopt `typer` for CLI command interface (if CLI ergonomics matter) @@ -39,13 +42,19 @@ This document outlines general practices and expectations for AI agents assistin * Use latest CDN version for embedded dashboards * Charts should be rendered from pre-generated JSON blobs in `/json/` +### Tables: DataTables + +* Use DataTables via CDN for reports with `chart: table` +* Requires jQuery from a CDN +* Table data comes from the same `/json/` files as charts + ### Styling: Bulma CSS * Use via CDN or vendored minified copy (to keep reports fully static) * Stick to default components (columns, cards, buttons, etc.) * No JS dependencies from Bulma -### Icon Set: [Feather Icons (CC0)](https://feathericons.com/) +### Icon Set: [Free CC0 Icons (CC0)](https://cc0-icons.jonh.eu/) * License: MIT / CC0-like * Use SVG versions @@ -83,6 +92,14 @@ ngxstat/ If uncertain, the agent should prompt the human for clarification before making architectural assumptions. +## Testing + +Use `pytest` for automated tests. Run the suite from an activated virtual environment and ensure all tests pass before committing: + +```bash +pytest -q +``` + --- ## Future Capabilities @@ -100,3 +117,4 @@ As the project matures, agents may also: * **2025-07-17**: Initial version by Jordan + ChatGPT * **2025-07-17**: Expanded virtual environment usage guidance + diff --git a/README.md b/README.md index 1a8a307..ac601fc 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,16 @@ # ngxstat -Per-domain Nginx log analytics with hybrid static reports and live insights. -## Generating Reports +`ngxstat` is a lightweight log analytics toolkit for Nginx. It imports access +logs into an SQLite database and renders static dashboards so you can explore +per-domain metrics without running a heavy backend service. -Use the `generate_reports.py` script to build aggregated JSON and HTML files from `database/ngxstat.db`. +## Requirements -Create a virtual environment and install dependencies: +* Python 3.10+ +* Access to the Nginx log files (default: `/var/log/nginx`) + +The helper scripts create a virtual environment on first run, but you can also +set one up manually: ```bash python3 -m venv .venv @@ -13,67 +18,95 @@ source .venv/bin/activate pip install -r requirements.txt ``` -Then run one or more of the interval commands: - -```bash -python scripts/generate_reports.py hourly -python scripts/generate_reports.py daily -python scripts/generate_reports.py weekly -python scripts/generate_reports.py monthly -``` - -Reports are written under the `output/` directory. Each command updates the corresponding `.json` file and produces an HTML dashboard using Chart.js. - ## Importing Logs -Use the `run-import.sh` script to set up the Python environment if needed and import the latest Nginx log entries into `database/ngxstat.db`. +Run the importer to ingest new log entries into `database/ngxstat.db`: ```bash ./run-import.sh ``` -This script is suitable for cron jobs as it creates the virtual environment on first run, installs dependencies and reuses the environment on subsequent runs. +Rotated logs are processed in order and only entries newer than the last +imported timestamp are added. -The importer handles rotated logs in order from oldest to newest so entries are -processed exactly once. If you rerun the script, it only ingests records with a -timestamp newer than the latest one already stored in the database, preventing -duplicates. +## Generating Reports -## Cron Report Generation - -Use the `run-reports.sh` script to run all report intervals in one step. The script sets up the Python environment the same way as `run-import.sh`, making it convenient for automation via cron. +To build the HTML dashboard and JSON data files use `run-reports.sh` which runs +all intervals in one go: ```bash ./run-reports.sh ``` -Running this script will create or update the hourly, daily, weekly and monthly reports under `output/`. +The script calls `scripts/generate_reports.py` internally to create hourly, +daily, weekly and monthly reports, then writes analysis JSON files used by the +"Analysis" tab. Per-domain reports are written under `output/domains/` +alongside the aggregate data. Open `output/index.html` in a browser to view the +dashboard. -## Serving Reports with Nginx +If you prefer to run individual commands you can invoke the generator directly: -To expose the generated HTML dashboards and JSON files over HTTP you can use a -simple Nginx server block. Point the `root` directive to the repository's -`output/` directory and optionally restrict access to your local network. +```bash +python scripts/generate_reports.py hourly +python scripts/generate_reports.py daily --all-domains +``` + +## Analysis Helpers + +`run-analysis.sh` executes additional utilities that examine the database for +missing domains, caching opportunities and potential threats. The JSON output is +saved under `output/analysis` and appears in the "Analysis" tab. The +`run-reports.sh` script also generates these JSON files as part of the build. + +## UX Controls + +The dashboard defaults to a 7‑day window for time series. Your view preferences +persist locally in the browser under the `ngxstat-state-v2` key. Use the +"Reset view" button to clear saved state and restore defaults. + +```bash +./run-analysis.sh +``` + +## Serving the Reports + +The generated files are static. You can serve them with a simple Nginx block: ```nginx server { listen 80; server_name example.com; - - # Path to the generated reports root /path/to/ngxstat/output; location / { try_files $uri $uri/ =404; } - - # Allow access only from private networks - allow 192.0.0.0/8; - allow 10.0.0.0/8; - deny all; } ``` -With this configuration the generated static files are served directly by -Nginx while connections outside of `192.*` and `10.*` are denied. +Restrict access if the reports should not be public. +## Running Tests + +Install the development dependencies and execute the suite with `pytest`: + +```bash +pip install -r requirements.txt +pytest -q +``` + +All tests must pass before submitting changes. + +## Acknowledgements + +ngxstat uses the following third‑party resources: + +* [Chart.js](https://www.chartjs.org/) for charts +* [DataTables](https://datatables.net/) and [jQuery](https://jquery.com/) for table views +* [Bulma CSS](https://bulma.io/) for styling +* Icons from [Free CC0 Icons](https://cc0-icons.jonh.eu/) by Jon Hicks (CC0 / MIT) +* [Typer](https://typer.tiangolo.com/) for the command-line interface +* [Jinja2](https://palletsprojects.com/p/jinja/) for templating + +The project is licensed under the GPLv3. Icon assets remain in the public domain +via the CC0 license. diff --git a/reports.yml b/reports.yml new file mode 100644 index 0000000..709d686 --- /dev/null +++ b/reports.yml @@ -0,0 +1,213 @@ +- name: hits + label: Hits + icon: pulse + chart: line + bucket: time_bucket + bucket_label: Time + query: | + SELECT {bucket} AS time_bucket, + COUNT(*) AS value + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket + +- name: error_rate + label: Error Rate (%) + icon: file-alert + chart: line + bucket: time_bucket + bucket_label: Time + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN status BETWEEN 400 AND 599 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket + +- name: cache_status_breakdown + label: Cache Status + icon: archive + chart: polarArea + bucket: cache_status + bucket_label: Cache Status + query: | + SELECT cache_status AS cache_status, + COUNT(*) AS value + FROM logs + GROUP BY cache_status + ORDER BY value DESC + colors: + - "#3273dc" + - "#23d160" + - "#ffdd57" + - "#ff3860" + - "#7957d5" + - "#363636" + +- name: domain_traffic + label: Top Domains + icon: globe + chart: table + top_n: 50 + per_domain: false + bucket: domain + bucket_label: Domain + query: | + SELECT host AS domain, + COUNT(*) AS value + FROM logs + GROUP BY domain + ORDER BY value DESC + +- name: bytes_sent + label: Bytes Sent + icon: upload + chart: line + bucket: time_bucket + bucket_label: Time + query: | + SELECT {bucket} AS time_bucket, + SUM(bytes_sent) AS value + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket + +- name: top_paths + label: Top Paths + icon: map + chart: table + top_n: 50 + buckets: + - domain + - path + bucket_label: + - Domain + - Path + query: | + WITH paths AS ( + SELECT host AS domain, + substr(substr(request, instr(request, ' ') + 1), 1, + instr(substr(request, instr(request, ' ') + 1), ' ') - 1) AS path + FROM logs + ), ranked AS ( + SELECT domain, path, COUNT(*) AS value, + ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn + FROM paths + GROUP BY domain, path + ) + SELECT domain, path, value + FROM ranked + WHERE rn <= 20 + ORDER BY domain, value DESC + +- name: user_agents + label: User Agents + icon: user + chart: table + top_n: 50 + buckets: + - domain + - user_agent + bucket_label: + - Domain + - User Agent + query: | + WITH ua AS ( + SELECT host AS domain, user_agent + FROM logs + ), ranked AS ( + SELECT domain, user_agent, COUNT(*) AS value, + ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn + FROM ua + GROUP BY domain, user_agent + ) + SELECT domain, user_agent, value + FROM ranked + WHERE rn <= 20 + ORDER BY domain, value DESC + +- name: referrers + label: Referrers + icon: link + chart: table + top_n: 50 + buckets: + - domain + - referrer + bucket_label: + - Domain + - Referrer + query: | + WITH ref AS ( + SELECT host AS domain, referer AS referrer + FROM logs + ), ranked AS ( + SELECT domain, referrer, COUNT(*) AS value, + ROW_NUMBER() OVER (PARTITION BY domain ORDER BY COUNT(*) DESC) AS rn + FROM ref + GROUP BY domain, referrer + ) + SELECT domain, referrer, value + FROM ranked + WHERE rn <= 20 + ORDER BY domain, value DESC + +- name: status_distribution + label: HTTP Statuses + icon: server + chart: pie + bucket: status_group + bucket_label: Status + query: | + SELECT CASE + WHEN status BETWEEN 200 AND 299 THEN '2xx' + WHEN status BETWEEN 300 AND 399 THEN '3xx' + WHEN status BETWEEN 400 AND 499 THEN '4xx' + ELSE '5xx' + END AS status_group, + COUNT(*) AS value + FROM logs + GROUP BY status_group + ORDER BY status_group + colors: + - "#48c78e" + - "#209cee" + - "#ffdd57" + - "#f14668" + +# New time-series: status classes over time (stacked) +- name: status_classes_timeseries + label: Status Classes Over Time + icon: server + chart: stackedBar + bucket: time_bucket + bucket_label: Time + stacked: true + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS "2xx", + SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END) AS "3xx", + SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS "4xx", + SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS "5xx", + COUNT(*) AS total + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket + +# New time-series: cache status over time (compact Hit/Miss; exclude '-' by default) +- name: cache_status_timeseries + label: Cache Status Over Time + icon: archive + chart: stackedBar + bucket: time_bucket + bucket_label: Time + stacked: true + exclude_values: ["-"] + query: | + SELECT {bucket} AS time_bucket, + SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) AS hit, + SUM(CASE WHEN cache_status = 'MISS' THEN 1 ELSE 0 END) AS miss, + COUNT(*) AS total + FROM logs + GROUP BY time_bucket + ORDER BY time_bucket diff --git a/requirements.txt b/requirements.txt index 221e3c8..2678f7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ Flask # For optional lightweight API server # Linting / formatting (optional but recommended) black flake8 +PyYAML diff --git a/run-analysis.sh b/run-analysis.sh new file mode 100755 index 0000000..4149b9a --- /dev/null +++ b/run-analysis.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e + +# Prevent concurrent executions of this script. +LOCK_FILE="/tmp/$(basename "$0").lock" +if [ -e "$LOCK_FILE" ]; then + echo "[WARN] $(basename "$0") is already running (lock file present)." >&2 + exit 0 +fi +touch "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + +# Ensure virtual environment exists +if [ ! -d ".venv" ]; then + echo "[INFO] Creating virtual environment..." + python3 -m venv .venv + source .venv/bin/activate + echo "[INFO] Installing dependencies..." + pip install --upgrade pip + if [ -f requirements.txt ]; then + pip install -r requirements.txt + else + echo "[WARN] requirements.txt not found, skipping." + fi +else + echo "[INFO] Activating virtual environment..." + source .venv/bin/activate +fi + +# Run analysis helpers +echo "[INFO] Checking for missing domains..." +python -m scripts.analyze check-missing-domains + +echo "[INFO] Suggesting cache improvements..." +python -m scripts.analyze suggest-cache + +echo "[INFO] Detecting threats..." +python -m scripts.analyze detect-threats + +# Deactivate to keep cron environment clean +if type deactivate >/dev/null 2>&1; then + deactivate +fi diff --git a/run-import.sh b/run-import.sh index 22b4b31..3c79d35 100755 --- a/run-import.sh +++ b/run-import.sh @@ -1,6 +1,17 @@ #!/usr/bin/env bash set -e +# Prevent multiple simultaneous runs by using a lock file specific to this +# script. If the lock already exists, assume another instance is running and +# exit gracefully. +LOCK_FILE="/tmp/$(basename "$0").lock" +if [ -e "$LOCK_FILE" ]; then + echo "[WARN] $(basename "$0") is already running (lock file present)." >&2 + exit 0 +fi +touch "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + # Ensure virtual environment exists if [ ! -d ".venv" ]; then echo "[INFO] Creating virtual environment..." diff --git a/run-reports.sh b/run-reports.sh index 56f4c31..f7cffba 100755 --- a/run-reports.sh +++ b/run-reports.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash set -e +# Prevent concurrent executions of this script. +LOCK_FILE="/tmp/$(basename "$0").lock" +if [ -e "$LOCK_FILE" ]; then + echo "[WARN] $(basename "$0") is already running (lock file present)." >&2 + exit 0 +fi +touch "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + # Ensure virtual environment exists if [ ! -d ".venv" ]; then echo "[INFO] Creating virtual environment..." @@ -18,12 +27,27 @@ else source .venv/bin/activate fi -# Generate all reports -echo "[INFO] Generating reports..." -python scripts/generate_reports.py hourly -python scripts/generate_reports.py daily -python scripts/generate_reports.py weekly -python scripts/generate_reports.py monthly +# Generate reports for all domains combined +echo "[INFO] Generating aggregate reports..." +python -m scripts.generate_reports hourly +python -m scripts.generate_reports daily +python -m scripts.generate_reports weekly +python -m scripts.generate_reports monthly +python -m scripts.generate_reports global + +# Generate reports for each individual domain +echo "[INFO] Generating per-domain reports..." +python -m scripts.generate_reports hourly --all-domains +python -m scripts.generate_reports daily --all-domains +python -m scripts.generate_reports weekly --all-domains +python -m scripts.generate_reports monthly --all-domains + +# Generate analysis JSON +echo "[INFO] Generating analysis files..." +python -m scripts.generate_reports analysis + +# Generate root index +python -m scripts.generate_reports index # Deactivate to keep cron environment clean if type deactivate >/dev/null 2>&1; then diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..f4c57a1 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"Utility package for ngxstat scripts" diff --git a/scripts/analyze.py b/scripts/analyze.py new file mode 100644 index 0000000..9f49978 --- /dev/null +++ b/scripts/analyze.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Utility helpers for ad-hoc log analysis. + +This module exposes small helper functions to inspect the ``ngxstat`` SQLite +database. The intent is to allow quick queries from the command line or other +scripts without rewriting SQL each time. + +Examples +-------- +To list all domains present in the database:: + + python scripts/analyze.py domains + +The CLI is powered by :mod:`typer` and currently only offers a couple of +commands. More analysis routines can be added over time. +""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import List, Optional, Set +from datetime import datetime, timedelta + +import json + +import typer + +from scripts import nginx_config # noqa: F401 # imported for side effects/usage + +DB_PATH = Path("database/ngxstat.db") +ANALYSIS_DIR = Path("output/analysis") + +app = typer.Typer(help="Ad-hoc statistics queries") + + +def _connect() -> sqlite3.Connection: + """Return a new SQLite connection to :data:`DB_PATH`.""" + return sqlite3.connect(DB_PATH) + + +def load_domains_from_db() -> List[str]: + """Return a sorted list of distinct domains from the ``logs`` table.""" + conn = _connect() + cur = conn.cursor() + cur.execute("SELECT DISTINCT host FROM logs ORDER BY host") + domains = [row[0] for row in cur.fetchall()] + conn.close() + return domains + + +def get_hit_count(domain: Optional[str] = None) -> int: + """Return total request count. + + Parameters + ---------- + domain: + Optional domain to filter on. If ``None`` the count includes all logs. + """ + conn = _connect() + cur = conn.cursor() + if domain: + cur.execute("SELECT COUNT(*) FROM logs WHERE host = ?", (domain,)) + else: + cur.execute("SELECT COUNT(*) FROM logs") + count = cur.fetchone()[0] or 0 + conn.close() + return count + + +def get_cache_ratio(domain: Optional[str] = None) -> float: + """Return the percentage of requests served from cache.""" + conn = _connect() + cur = conn.cursor() + if domain: + cur.execute( + "SELECT SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) * 1.0 / " + "COUNT(*) FROM logs WHERE host = ?", + (domain,), + ) + else: + cur.execute( + "SELECT SUM(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) * 1.0 / " + "COUNT(*) FROM logs" + ) + result = cur.fetchone()[0] + conn.close() + return float(result or 0.0) + + +@app.command() +def domains() -> None: + """Print the list of domains discovered in the database.""" + for d in load_domains_from_db(): + typer.echo(d) + + +@app.command() +def hits(domain: Optional[str] = typer.Option(None, help="Filter by domain")) -> None: + """Show request count.""" + count = get_hit_count(domain) + if domain: + typer.echo(f"{domain}: {count} hits") + else: + typer.echo(f"Total hits: {count}") + + +@app.command("cache-ratio") +def cache_ratio_cmd( + domain: Optional[str] = typer.Option(None, help="Filter by domain") +) -> None: + """Display cache hit ratio as a percentage.""" + ratio = get_cache_ratio(domain) * 100 + if domain: + typer.echo(f"{domain}: {ratio:.2f}% cached") + else: + typer.echo(f"Cache hit ratio: {ratio:.2f}%") + + +@app.command("check-missing-domains") +def check_missing_domains( + json_output: bool = typer.Option( + False, "--json", help="Output missing domains as JSON" + ) +) -> None: + """Show domains present in the database but absent from Nginx config.""" + try: + from scripts.generate_reports import _get_domains as _db_domains + except Exception: # pragma: no cover - fallback if import fails + _db_domains = load_domains_from_db + + if not isinstance(json_output, bool): + json_output = False + + db_domains = set(_db_domains()) + + paths = nginx_config.discover_configs() + servers = nginx_config.parse_servers(paths) + config_domains: Set[str] = set() + for server in servers: + names = server.get("server_name", "") + for name in names.split(): + if name: + config_domains.add(name) + + missing = sorted(db_domains - config_domains) + + ANALYSIS_DIR.mkdir(parents=True, exist_ok=True) + out_path = ANALYSIS_DIR / "missing_domains.json" + out_path.write_text(json.dumps(missing, indent=2)) + + if json_output: + typer.echo(json.dumps(missing)) + else: + for d in missing: + typer.echo(d) + + +def suggest_cache( + threshold: int = 10, + json_output: bool = False, +) -> None: + """Suggest domain/path pairs that could benefit from caching. + + Paths with at least ``threshold`` ``MISS`` entries are shown for domains + whose server blocks lack a ``proxy_cache`` directive. + """ + + # Discover domains without explicit proxy_cache + paths = nginx_config.discover_configs() + servers = nginx_config.parse_servers(paths) + no_cache: Set[str] = set() + for server in servers: + if "proxy_cache" in server: + continue + for name in server.get("server_name", "").split(): + if name: + no_cache.add(name) + + conn = _connect() + cur = conn.cursor() + cur.execute( + """ + SELECT host, + substr(request, instr(request, ' ')+1, + instr(request, ' HTTP') - instr(request, ' ') - 1) AS path, + COUNT(*) AS miss_count + FROM logs + WHERE cache_status = 'MISS' + GROUP BY host, path + HAVING miss_count >= ? + ORDER BY miss_count DESC + """, + (int(threshold),), + ) + + rows = [r for r in cur.fetchall() if r[0] in no_cache] + conn.close() + + result = [ + {"host": host, "path": path, "misses": count} for host, path, count in rows + ] + + ANALYSIS_DIR.mkdir(parents=True, exist_ok=True) + out_path = ANALYSIS_DIR / "cache_suggestions.json" + out_path.write_text(json.dumps(result, indent=2)) + + if json_output: + typer.echo(json.dumps(result)) + else: + for item in result: + typer.echo(f"{item['host']} {item['path']} {item['misses']}") + +@app.command("suggest-cache") +def suggest_cache_cli( + threshold: int = typer.Option(10, help="Minimum number of MISS entries to report"), + json_output: bool = typer.Option(False, "--json", help="Output results as JSON"), +) -> None: + """CLI wrapper for suggest_cache.""" + suggest_cache(threshold=threshold, json_output=json_output) + + +def detect_threats( + hours: int = 1, + ip_threshold: int = 100, +) -> None: + """Detect potential security threats from recent logs.""" + + conn = _connect() + cur = conn.cursor() + + cur.execute("SELECT MAX(time) FROM logs") + row = cur.fetchone() + if not row or not row[0]: + typer.echo("No logs found") + conn.close() + return + + max_dt = datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") + recent_end = max_dt + recent_start = recent_end - timedelta(hours=int(hours)) + prev_start = recent_start - timedelta(hours=int(hours)) + prev_end = recent_start + + fmt = "%Y-%m-%d %H:%M:%S" + recent_start_s = recent_start.strftime(fmt) + recent_end_s = recent_end.strftime(fmt) + prev_start_s = prev_start.strftime(fmt) + prev_end_s = prev_end.strftime(fmt) + + cur.execute( + """ + SELECT host, + SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) AS errors, + COUNT(*) AS total + FROM logs + WHERE time >= ? AND time < ? + GROUP BY host + """, + (recent_start_s, recent_end_s), + ) + recent_rows = {r[0]: (r[1], r[2]) for r in cur.fetchall()} + + cur.execute( + """ + SELECT host, + SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) AS errors, + COUNT(*) AS total + FROM logs + WHERE time >= ? AND time < ? + GROUP BY host + """, + (prev_start_s, prev_end_s), + ) + prev_rows = {r[0]: (r[1], r[2]) for r in cur.fetchall()} + + error_spikes = [] + for host in set(recent_rows) | set(prev_rows): + r_err, r_total = recent_rows.get(host, (0, 0)) + p_err, p_total = prev_rows.get(host, (0, 0)) + r_rate = r_err * 100.0 / r_total if r_total else 0.0 + p_rate = p_err * 100.0 / p_total if p_total else 0.0 + if r_rate >= 10 and r_rate >= p_rate * 2: + error_spikes.append( + { + "host": host, + "recent_error_rate": round(r_rate, 2), + "previous_error_rate": round(p_rate, 2), + } + ) + + cur.execute( + """ + SELECT DISTINCT user_agent FROM logs + WHERE time >= ? AND time < ? + """, + (prev_start_s, prev_end_s), + ) + prev_agents = {r[0] for r in cur.fetchall()} + + cur.execute( + """ + SELECT user_agent, COUNT(*) AS c + FROM logs + WHERE time >= ? AND time < ? + GROUP BY user_agent + HAVING c >= 10 + """, + (recent_start_s, recent_end_s), + ) + suspicious_agents = [ + {"user_agent": ua, "requests": cnt} + for ua, cnt in cur.fetchall() + if ua not in prev_agents + ] + + cur.execute( + """ + SELECT ip, COUNT(*) AS c + FROM logs + WHERE time >= ? AND time < ? + GROUP BY ip + HAVING c >= ? + ORDER BY c DESC + """, + (recent_start_s, recent_end_s, ip_threshold), + ) + high_ip_requests = [{"ip": ip, "requests": cnt} for ip, cnt in cur.fetchall()] + + conn.close() + + report = { + "time_range": { + "recent_start": recent_start_s, + "recent_end": recent_end_s, + "previous_start": prev_start_s, + "previous_end": prev_end_s, + }, + "error_spikes": error_spikes, + "suspicious_agents": suspicious_agents, + "high_ip_requests": high_ip_requests, + } + + ANALYSIS_DIR.mkdir(parents=True, exist_ok=True) + out_path = ANALYSIS_DIR / "threat_report.json" + out_path.write_text(json.dumps(report, indent=2)) + typer.echo(json.dumps(report)) + +@app.command("detect-threats") +def detect_threats_cli( + hours: int = typer.Option(1, help="Number of recent hours to analyze"), + ip_threshold: int = typer.Option(100, help="Requests from a single IP to flag"), +) -> None: + """CLI wrapper for detect_threats.""" + detect_threats(hours=hours, ip_threshold=ip_threshold) + + +if __name__ == "__main__": + app() diff --git a/scripts/download_icons.py b/scripts/download_icons.py new file mode 100644 index 0000000..6f4675a --- /dev/null +++ b/scripts/download_icons.py @@ -0,0 +1,28 @@ +import json +from urllib.request import urlopen, Request +from pathlib import Path + +ICON_LIST_URL = "https://cc0-icons.jonh.eu/icons.json" +BASE_URL = "https://cc0-icons.jonh.eu/" + +OUTPUT_DIR = Path(__file__).resolve().parent.parent / "static" / "icons" + + +def main() -> None: + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + req = Request(ICON_LIST_URL, headers={"User-Agent": "Mozilla/5.0"}) + with urlopen(req) as resp: + data = json.load(resp) + icons = data.get("icons", []) + for icon in icons: + slug = icon.get("slug") + url = BASE_URL + icon.get("url") + path = OUTPUT_DIR / f"{slug}.svg" + req = Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urlopen(req) as resp: + path.write_bytes(resp.read()) + print(f"Downloaded {len(icons)} icons to {OUTPUT_DIR}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_reports.py b/scripts/generate_reports.py index b244075..d3c2f8a 100644 --- a/scripts/generate_reports.py +++ b/scripts/generate_reports.py @@ -1,79 +1,466 @@ import json +import sys import sqlite3 from pathlib import Path -from typing import List, Dict +import shutil +from typing import List, Dict, Optional +from datetime import datetime, timezone +import time + +import yaml import typer from jinja2 import Environment, FileSystemLoader +# Ensure project root is importable when running as a script (python scripts/generate_reports.py) +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + DB_PATH = Path("database/ngxstat.db") OUTPUT_DIR = Path("output") TEMPLATE_DIR = Path("templates") +REPORT_CONFIG = Path("reports.yml") +GENERATED_MARKER = OUTPUT_DIR / "generated.txt" + +# Mapping of interval names to SQLite strftime formats. These strings are +# substituted into report queries whenever the special ``{bucket}`` token is +# present so that a single report definition can be reused for multiple +# intervals. +INTERVAL_FORMATS = { + "hourly": "%Y-%m-%d %H:00:00", + "daily": "%Y-%m-%d", + "weekly": "%Y-%W", + "monthly": "%Y-%m", +} app = typer.Typer(help="Generate aggregated log reports") -def _load_existing(path: Path) -> List[Dict]: - if path.exists(): - try: - return json.loads(path.read_text()) - except Exception: - return [] - return [] + +@app.callback() +def _cli_callback(ctx: typer.Context) -> None: + """Register post-command hook to note generation time.""" + + def _write_marker() -> None: + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + # Use timezone-aware UTC to avoid deprecation warnings and ambiguity + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + GENERATED_MARKER.write_text(f"{timestamp}\n") + + ctx.call_on_close(_write_marker) + + +def _get_domains() -> List[str]: + """Return a sorted list of unique domains from the logs table.""" + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + cur.execute("SELECT DISTINCT host FROM logs ORDER BY host") + domains = [row[0] for row in cur.fetchall()] + conn.close() + return domains + + +def _load_config() -> List[Dict]: + if not REPORT_CONFIG.exists(): + typer.echo(f"Config file not found: {REPORT_CONFIG}") + raise typer.Exit(1) + with REPORT_CONFIG.open("r") as fh: + data = yaml.safe_load(fh) or [] + if not isinstance(data, list): + typer.echo("reports.yml must contain a list of report definitions") + raise typer.Exit(1) + return data + def _save_json(path: Path, data: List[Dict]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2)) -def _render_html(interval: str, json_name: str, out_path: Path) -> None: + +def _copy_icons() -> None: + """Copy vendored icons and scripts to the output directory.""" + src_dir = Path("static/icons") + dst_dir = OUTPUT_DIR / "icons" + if src_dir.is_dir(): + dst_dir.mkdir(parents=True, exist_ok=True) + for icon in src_dir.glob("*.svg"): + shutil.copy(icon, dst_dir / icon.name) + + js_src = Path("static/chartManager.js") + if js_src.is_file(): + shutil.copy(js_src, OUTPUT_DIR / js_src.name) + + +def _render_snippet(report: Dict, out_dir: Path) -> None: + """Render a single report snippet to ``.html`` inside ``out_dir``.""" env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) - template = env.get_template("report.html") - out_path.write_text(template.render(interval=interval, json_path=json_name)) + template = env.get_template("report_snippet.html") + snippet_path = out_dir / f"{report['name']}.html" + snippet_path.write_text(template.render(report=report)) -def _aggregate(interval: str, fmt: str) -> None: - json_path = OUTPUT_DIR / f"{interval}.json" - html_path = OUTPUT_DIR / f"{interval}.html" - existing = _load_existing(json_path) - last_bucket = existing[-1]["bucket"] if existing else None +def _write_stats( + generated_at: Optional[str] = None, generation_seconds: Optional[float] = None +) -> None: + """Query basic dataset stats and write them to ``output/global/stats.json``.""" + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + cur.execute("SELECT COUNT(*) FROM logs") + total_logs = cur.fetchone()[0] or 0 + + cur.execute("SELECT MIN(time), MAX(time) FROM logs") + row = cur.fetchone() or (None, None) + start_date = row[0] or "" + end_date = row[1] or "" + + cur.execute("SELECT COUNT(DISTINCT host) FROM logs") + unique_domains = cur.fetchone()[0] or 0 + + conn.close() + + stats = { + "total_logs": total_logs, + "start_date": start_date, + "end_date": end_date, + "unique_domains": unique_domains, + } + if generated_at: + stats["generated_at"] = generated_at + if generation_seconds is not None: + stats["generation_seconds"] = generation_seconds + + out_path = OUTPUT_DIR / "global" / "stats.json" + _save_json(out_path, stats) + + +def _bucket_expr(interval: str) -> str: + """Return the SQLite strftime expression for the given interval.""" + fmt = INTERVAL_FORMATS.get(interval) + if not fmt: + typer.echo(f"Unsupported interval: {interval}") + raise typer.Exit(1) + return f"strftime('{fmt}', datetime(time))" + + +def _generate_interval(interval: str, domain: Optional[str] = None) -> None: + cfg = _load_config() + if not cfg: + typer.echo("No report definitions found") + return + + _copy_icons() + + bucket = _bucket_expr(interval) conn = sqlite3.connect(DB_PATH) cur = conn.cursor() - query = f"SELECT strftime('{fmt}', datetime(time)) as bucket, COUNT(*) as hits FROM logs" - params = [] - if last_bucket: - query += " WHERE datetime(time) > datetime(?)" - params.append(last_bucket) - query += " GROUP BY bucket ORDER BY bucket" + # Create a temporary view so queries can easily be filtered by domain + cur.execute("DROP VIEW IF EXISTS logs_view") + if domain: + # Parameters are not allowed in CREATE VIEW statements, so we must + # safely interpolate the domain value ourselves. Escape any single + # quotes to prevent malformed queries. + safe_domain = domain.replace("'", "''") + cur.execute( + f"CREATE TEMP VIEW logs_view AS SELECT * FROM logs WHERE host = '{safe_domain}'" + ) + out_dir = OUTPUT_DIR / "domains" / domain / interval + else: + cur.execute("CREATE TEMP VIEW logs_view AS SELECT * FROM logs") + out_dir = OUTPUT_DIR / interval - rows = cur.execute(query, params).fetchall() - for bucket, hits in rows: - existing.append({"bucket": bucket, "hits": hits}) + out_dir.mkdir(parents=True, exist_ok=True) + + report_list = [] + for definition in cfg: + if "{bucket}" not in definition["query"] or definition.get("global"): + # Global reports are generated separately + continue + if domain and not definition.get("per_domain", True): + # Skip reports marked as not applicable to per-domain runs + continue + + name = definition["name"] + query = definition["query"].replace("{bucket}", bucket) + query = query.replace("FROM logs", "FROM logs_view") + # Apply top_n limit for tables (performance-friendly), if configured + top_n = definition.get("top_n") + chart_type = definition.get("chart", "line") + if top_n and chart_type == "table": + try: + n = int(top_n) + if "LIMIT" not in query.upper(): + query = f"{query}\nLIMIT {n}" + except Exception: + pass + cur.execute(query) + rows = cur.fetchall() + headers = [c[0] for c in cur.description] + data = [dict(zip(headers, row)) for row in rows] + json_path = out_dir / f"{name}.json" + _save_json(json_path, data) + entry = { + "name": name, + "label": definition.get("label", name.title()), + "chart": definition.get("chart", "line"), + "json": f"{name}.json", + "html": f"{name}.html", + } + if "icon" in definition: + entry["icon"] = definition["icon"] + if "bucket" in definition: + entry["bucket"] = definition["bucket"] + if "buckets" in definition: + entry["buckets"] = definition["buckets"] + if "bucket_label" in definition: + entry["bucket_label"] = definition["bucket_label"] + if "color" in definition: + entry["color"] = definition["color"] + if "colors" in definition: + entry["colors"] = definition["colors"] + # Optional UX metadata passthrough for frontend-only transforms + for key in ( + "windows_supported", + "window_default", + "group_others_threshold", + "exclude_values", + "top_n", + "stacked", + "palette", + ): + if key in definition: + entry[key] = definition[key] + _render_snippet(entry, out_dir) + report_list.append(entry) + + _save_json(out_dir / "reports.json", report_list) + if domain: + typer.echo(f"Generated {interval} reports for {domain}") + else: + typer.echo(f"Generated {interval} reports") + + +def _generate_all_domains(interval: str) -> None: + """Generate reports for each unique domain.""" + for domain in _get_domains(): + _generate_interval(interval, domain) + + +def _generate_root_index() -> None: + """Render the top-level index listing all intervals and domains.""" + _copy_icons() + intervals = sorted( + [name for name in INTERVAL_FORMATS if (OUTPUT_DIR / name).is_dir()] + ) + + domains_dir = OUTPUT_DIR / "domains" + domains: List[str] = [] + if domains_dir.is_dir(): + domains = [p.name for p in domains_dir.iterdir() if p.is_dir()] + domains.sort() + + env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) + template = env.get_template("index.html") + + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + out_path = OUTPUT_DIR / "index.html" + out_path.write_text(template.render(intervals=intervals, domains=domains)) + typer.echo(f"Generated root index at {out_path}") + + +def _generate_global() -> None: + """Generate reports that do not depend on an interval.""" + cfg = _load_config() + if not cfg: + typer.echo("No report definitions found") + return + + start_time = time.time() + # Use timezone-aware UTC for generated_at (string remains unchanged format) + generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + _copy_icons() + + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + out_dir = OUTPUT_DIR / "global" + out_dir.mkdir(parents=True, exist_ok=True) + + report_list = [] + for definition in cfg: + if "{bucket}" in definition["query"] and not definition.get("global"): + continue + + name = definition["name"] + query = definition["query"] + # Apply top_n limit for tables (performance-friendly), if configured + top_n = definition.get("top_n") + chart_type = definition.get("chart", "line") + if top_n and chart_type == "table": + try: + n = int(top_n) + if "LIMIT" not in query.upper(): + query = f"{query}\nLIMIT {n}" + except Exception: + pass + cur.execute(query) + rows = cur.fetchall() + headers = [c[0] for c in cur.description] + data = [dict(zip(headers, row)) for row in rows] + json_path = out_dir / f"{name}.json" + _save_json(json_path, data) + entry = { + "name": name, + "label": definition.get("label", name.title()), + "chart": definition.get("chart", "line"), + "json": f"{name}.json", + "html": f"{name}.html", + } + if "icon" in definition: + entry["icon"] = definition["icon"] + if "bucket" in definition: + entry["bucket"] = definition["bucket"] + if "buckets" in definition: + entry["buckets"] = definition["buckets"] + if "bucket_label" in definition: + entry["bucket_label"] = definition["bucket_label"] + if "color" in definition: + entry["color"] = definition["color"] + if "colors" in definition: + entry["colors"] = definition["colors"] + # Optional UX metadata passthrough for frontend-only transforms + for key in ( + "windows_supported", + "window_default", + "group_others_threshold", + "exclude_values", + "top_n", + "stacked", + "palette", + ): + if key in definition: + entry[key] = definition[key] + _render_snippet(entry, out_dir) + report_list.append(entry) + + _save_json(out_dir / "reports.json", report_list) + elapsed = round(time.time() - start_time, 2) + _write_stats(generated_at, elapsed) + typer.echo("Generated global reports") + + +def _generate_analysis() -> None: + """Generate analysis JSON files consumed by the Analysis tab.""" + try: + # Import lazily to avoid circulars and keep dependencies optional + from scripts import analyze + except Exception as exc: # pragma: no cover - defensive + typer.echo(f"Failed to import analysis module: {exc}") + return + + # Ensure output root and icons present for parity + _copy_icons() + + # These commands write JSON files under output/analysis/ + try: + analyze.check_missing_domains(json_output=True) + except Exception as exc: # pragma: no cover - continue best-effort + typer.echo(f"check_missing_domains failed: {exc}") + try: + analyze.suggest_cache(json_output=True) + except Exception as exc: # pragma: no cover + typer.echo(f"suggest_cache failed: {exc}") + try: + analyze.detect_threats() + except Exception as exc: # pragma: no cover + typer.echo(f"detect_threats failed: {exc}") + typer.echo("Generated analysis JSON files") - existing.sort(key=lambda x: x["bucket"]) - _save_json(json_path, existing) - _render_html(interval, json_path.name, html_path) - typer.echo(f"Generated {json_path} and {html_path}") @app.command() -def hourly() -> None: - """Aggregate logs into hourly buckets.""" - _aggregate("hourly", "%Y-%m-%d %H:00:00") +def hourly( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: + """Generate hourly reports.""" + if all_domains: + _generate_all_domains("hourly") + else: + _generate_interval("hourly", domain) + @app.command() -def daily() -> None: - """Aggregate logs into daily buckets.""" - _aggregate("daily", "%Y-%m-%d") +def daily( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: + """Generate daily reports.""" + if all_domains: + _generate_all_domains("daily") + else: + _generate_interval("daily", domain) + @app.command() -def weekly() -> None: - """Aggregate logs into weekly buckets.""" - _aggregate("weekly", "%Y-%W") +def weekly( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: + """Generate weekly reports.""" + if all_domains: + _generate_all_domains("weekly") + else: + _generate_interval("weekly", domain) + @app.command() -def monthly() -> None: - """Aggregate logs into monthly buckets.""" - _aggregate("monthly", "%Y-%m") +def monthly( + domain: Optional[str] = typer.Option( + None, help="Generate reports for a specific domain" + ), + all_domains: bool = typer.Option( + False, "--all-domains", help="Generate reports for each domain" + ), +) -> None: + """Generate monthly reports.""" + if all_domains: + _generate_all_domains("monthly") + else: + _generate_interval("monthly", domain) + + +@app.command("global") +def global_reports() -> None: + """Generate global reports.""" + _generate_global() + + +@app.command() +def analysis() -> None: + """Generate analysis JSON files for the Analysis tab.""" + _generate_analysis() + + +@app.command() +def index() -> None: + """Generate the root index page linking all reports.""" + _generate_root_index() + if __name__ == "__main__": app() diff --git a/scripts/init_db.py b/scripts/init_db.py index 8bc70fb..b9ea07d 100644 --- a/scripts/init_db.py +++ b/scripts/init_db.py @@ -3,7 +3,7 @@ import os import re import sqlite3 -from datetime import datetime +from datetime import datetime, timezone LOG_DIR = "/var/log/nginx" DB_FILE = "database/ngxstat.db" @@ -42,10 +42,16 @@ cursor.execute("SELECT time FROM logs ORDER BY id DESC LIMIT 1") row = cursor.fetchone() last_dt = None if row and row[0]: - try: - last_dt = datetime.strptime(row[0], DATE_FMT) - except ValueError: - last_dt = None + # Support both legacy log date format and ISO timestamps + for fmt in ("%Y-%m-%d %H:%M:%S", DATE_FMT): + try: + parsed = datetime.strptime(row[0], fmt) + if fmt == DATE_FMT: + parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) + last_dt = parsed + break + except ValueError: + continue try: log_files = [] @@ -55,7 +61,9 @@ try: suffix = match.group(1) number = int(suffix.lstrip(".")) if suffix else 0 log_files.append((number, os.path.join(LOG_DIR, f))) - log_files = [path for _, path in sorted(log_files, key=lambda x: x[0], reverse=True)] + log_files = [ + path for _, path in sorted(log_files, key=lambda x: x[0], reverse=True) + ] except FileNotFoundError: print(f"[ERROR] Log directory not found: {LOG_DIR}") exit(1) @@ -74,6 +82,7 @@ for log_file in log_files: entry_dt = datetime.strptime(data["time"], DATE_FMT) except ValueError: continue + entry_dt = entry_dt.astimezone(timezone.utc).replace(tzinfo=None) if last_dt and entry_dt <= last_dt: continue cursor.execute( @@ -86,7 +95,7 @@ for log_file in log_files: ( data["ip"], data["host"], - data["time"], + entry_dt.strftime("%Y-%m-%d %H:%M:%S"), data["request"], int(data["status"]), int(data["bytes_sent"]), diff --git a/scripts/nginx_config.py b/scripts/nginx_config.py new file mode 100644 index 0000000..bc585a7 --- /dev/null +++ b/scripts/nginx_config.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Utilities for discovering and parsing Nginx configuration files. + +This module provides helper functions to locate Nginx configuration files and +extract key details from ``server`` blocks. Typical usage:: + + from scripts.nginx_config import discover_configs, parse_servers + + files = discover_configs() + servers = parse_servers(files) + for s in servers: + print(s.get("server_name"), s.get("listen")) + +The functions intentionally tolerate missing or unreadable files and will simply +skip over them. +""" + +import os +import re +from pathlib import Path +from typing import Dict, List, Set + +DEFAULT_PATHS = [ + "/etc/nginx/nginx.conf", + "/usr/local/etc/nginx/nginx.conf", +] + +INCLUDE_RE = re.compile(r"^\s*include\s+(.*?);", re.MULTILINE) +SERVER_RE = re.compile(r"server\s*{(.*?)}", re.DOTALL) +DIRECTIVE_RE = re.compile(r"^\s*(\S+)\s+(.*?);", re.MULTILINE) + + +def discover_configs() -> Set[Path]: + """Return a set of all config files reachable from :data:`DEFAULT_PATHS`.""" + + found: Set[Path] = set() + queue = [Path(p) for p in DEFAULT_PATHS] + + while queue: + path = queue.pop() + if path in found: + continue + if not path.exists(): + continue + try: + text = path.read_text() + except OSError: + continue + found.add(path) + for pattern in INCLUDE_RE.findall(text): + pattern = os.path.expanduser(pattern.strip()) + if os.path.isabs(pattern): + # ``Path.glob`` does not allow absolute patterns, so we + # anchor at the filesystem root and remove the leading + # separator. + base = Path(os.sep) + glob_iter = base.glob(pattern.lstrip(os.sep)) + else: + glob_iter = path.parent.glob(pattern) + for included in glob_iter: + if included.is_file() and included not in found: + queue.append(included) + return found + + +def parse_servers(paths: Set[Path]) -> List[Dict[str, str]]: + """Parse ``server`` blocks from the given files. + + Parameters + ---------- + paths: + Iterable of configuration file paths. + """ + + servers: List[Dict[str, str]] = [] + for p in paths: + try: + text = Path(p).read_text() + except OSError: + continue + for block in SERVER_RE.findall(text): + directives: Dict[str, List[str]] = {} + for name, value in DIRECTIVE_RE.findall(block): + directives.setdefault(name, []).append(value.strip()) + entry: Dict[str, str] = {} + if "server_name" in directives: + entry["server_name"] = " ".join(directives["server_name"]) + if "listen" in directives: + entry["listen"] = " ".join(directives["listen"]) + if "proxy_cache" in directives: + entry["proxy_cache"] = " ".join(directives["proxy_cache"]) + if "root" in directives: + entry["root"] = " ".join(directives["root"]) + servers.append(entry) + return servers diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..be5087c --- /dev/null +++ b/setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -e + +# Default schedules +import_sched="*/5 * * * *" +report_sched="0 * * * *" +analysis_sched="0 0 * * *" +remove=false + +usage() { + echo "Usage: $0 [--import CRON] [--reports CRON] [--analysis CRON] [--remove]" +} + +while [ $# -gt 0 ]; do + case "$1" in + --import) + import_sched="$2"; shift 2;; + --reports) + report_sched="$2"; shift 2;; + --analysis) + analysis_sched="$2"; shift 2;; + --remove) + remove=true; shift;; + -h|--help) + usage; exit 0;; + *) + usage; exit 1;; + esac +done + +repo_dir="$(cd "$(dirname "$0")" && pwd)" + +if [ "$remove" = true ]; then + tmp=$(mktemp) + sudo crontab -l 2>/dev/null | grep -v "# ngxstat import" | grep -v "# ngxstat reports" | grep -v "# ngxstat analysis" > "$tmp" || true + sudo crontab "$tmp" + rm -f "$tmp" + echo "[INFO] Removed ngxstat cron entries" + exit 0 +fi + +cron_entries="${import_sched} cd ${repo_dir} && ./run-import.sh # ngxstat import\n${report_sched} cd ${repo_dir} && ./run-reports.sh # ngxstat reports\n${analysis_sched} cd ${repo_dir} && ./run-analysis.sh # ngxstat analysis" + +( sudo crontab -l 2>/dev/null; echo -e "$cron_entries" ) | sudo crontab - + +echo "[INFO] Installed ngxstat cron entries" diff --git a/static/chartManager.js b/static/chartManager.js new file mode 100644 index 0000000..2f14f4f --- /dev/null +++ b/static/chartManager.js @@ -0,0 +1,109 @@ +export let currentLoad = null; +const loadInfo = new Map(); + +export function newLoad(container) { + if (currentLoad) { + abortLoad(currentLoad); + } + reset(container); + const controller = new AbortController(); + const token = { controller, charts: new Map() }; + loadInfo.set(token, token); + currentLoad = token; + return token; +} + +export function abortLoad(token) { + const info = loadInfo.get(token); + if (!info) return; + info.controller.abort(); + info.charts.forEach(chart => { + try { + chart.destroy(); + } catch (e) {} + }); + loadInfo.delete(token); + if (currentLoad === token) { + currentLoad = null; + } +} + +export function registerChart(token, id, chart) { + const info = loadInfo.get(token); + if (info) { + info.charts.set(id, chart); + } else { + chart.destroy(); + } +} + +export function reset(container) { + if (!container) return; + container.querySelectorAll('canvas').forEach(c => { + const chart = Chart.getChart(c); + if (chart) { + chart.destroy(); + } + }); + container.innerHTML = ''; +} + +// ---- Lightweight client-side data helpers ---- + +// Slice last N rows from a time-ordered array +export function sliceWindow(data, n) { + if (!Array.isArray(data) || n === undefined || n === null) return data; + if (n === 'all') return data; + const count = Number(n); + if (!Number.isFinite(count) || count <= 0) return data; + return data.slice(-count); +} + +// Exclude rows whose value in key is in excluded list +export function excludeValues(data, key, excluded = []) { + if (!excluded || excluded.length === 0) return data; + const set = new Set(excluded); + return data.filter(row => !set.has(row[key])); +} + +// Compute percentages for categorical distributions (valueKey default 'value') +export function toPercent(data, valueKey = 'value') { + const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0); + if (total <= 0) return data.map(r => ({ ...r })); + return data.map(r => ({ ...r, [valueKey]: (Number(r[valueKey]) || 0) * 100 / total })); +} + +// Group categories with share < threshold into an 'Other' bucket. +export function groupOthers(data, bucketKey, valueKey = 'value', threshold = 0.03, otherLabel = 'Other') { + if (!Array.isArray(data) || data.length === 0) return data; + const total = data.reduce((s, r) => s + (Number(r[valueKey]) || 0), 0); + if (total <= 0) return data; + const major = []; + let other = 0; + for (const r of data) { + const v = Number(r[valueKey]) || 0; + if (total && v / total < threshold) { + other += v; + } else { + major.push({ ...r }); + } + } + if (other > 0) major.push({ [bucketKey]: otherLabel, [valueKey]: other }); + return major; +} + +// Simple moving average over numeric array +export function movingAverage(series, span = 3) { + const n = Math.max(1, Number(span) || 1); + const out = []; + for (let i = 0; i < series.length; i++) { + const start = Math.max(0, i - n + 1); + let sum = 0, cnt = 0; + for (let j = start; j <= i; j++) { + const v = Number(series[j]); + if (Number.isFinite(v)) { sum += v; cnt++; } + } + out.push(cnt ? sum / cnt : null); + } + return out; +} diff --git a/static/icons/achievement.svg b/static/icons/achievement.svg new file mode 100644 index 0000000..a45ed68 --- /dev/null +++ b/static/icons/achievement.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/align-center.svg b/static/icons/align-center.svg new file mode 100644 index 0000000..63f0ce2 --- /dev/null +++ b/static/icons/align-center.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/align-left.svg b/static/icons/align-left.svg new file mode 100644 index 0000000..1e9144d --- /dev/null +++ b/static/icons/align-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/align-right.svg b/static/icons/align-right.svg new file mode 100644 index 0000000..d854828 --- /dev/null +++ b/static/icons/align-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/app-marketplace.svg b/static/icons/app-marketplace.svg new file mode 100644 index 0000000..e833972 --- /dev/null +++ b/static/icons/app-marketplace.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/archive.svg b/static/icons/archive.svg new file mode 100644 index 0000000..e8df12a --- /dev/null +++ b/static/icons/archive.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-diagonal-2.svg b/static/icons/arrow-double-diagonal-2.svg new file mode 100644 index 0000000..7a32329 --- /dev/null +++ b/static/icons/arrow-double-diagonal-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-diagonal-3.svg b/static/icons/arrow-double-diagonal-3.svg new file mode 100644 index 0000000..3c86255 --- /dev/null +++ b/static/icons/arrow-double-diagonal-3.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-diagonal-4.svg b/static/icons/arrow-double-diagonal-4.svg new file mode 100644 index 0000000..9c8beaa --- /dev/null +++ b/static/icons/arrow-double-diagonal-4.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-diagonal.svg b/static/icons/arrow-double-diagonal.svg new file mode 100644 index 0000000..f3c93f4 --- /dev/null +++ b/static/icons/arrow-double-diagonal.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-horizontal-2.svg b/static/icons/arrow-double-horizontal-2.svg new file mode 100644 index 0000000..f8be44b --- /dev/null +++ b/static/icons/arrow-double-horizontal-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-horizontal.svg b/static/icons/arrow-double-horizontal.svg new file mode 100644 index 0000000..d06fc55 --- /dev/null +++ b/static/icons/arrow-double-horizontal.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-vertical-2.svg b/static/icons/arrow-double-vertical-2.svg new file mode 100644 index 0000000..5f81dba --- /dev/null +++ b/static/icons/arrow-double-vertical-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-double-vertical.svg b/static/icons/arrow-double-vertical.svg new file mode 100644 index 0000000..ffd66ce --- /dev/null +++ b/static/icons/arrow-double-vertical.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down-2.svg b/static/icons/arrow-down-2.svg new file mode 100644 index 0000000..9bb9519 --- /dev/null +++ b/static/icons/arrow-down-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down-left-2.svg b/static/icons/arrow-down-left-2.svg new file mode 100644 index 0000000..c7e116f --- /dev/null +++ b/static/icons/arrow-down-left-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down-left.svg b/static/icons/arrow-down-left.svg new file mode 100644 index 0000000..edbc82c --- /dev/null +++ b/static/icons/arrow-down-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down-right-2.svg b/static/icons/arrow-down-right-2.svg new file mode 100644 index 0000000..5f59d55 --- /dev/null +++ b/static/icons/arrow-down-right-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down-right.svg b/static/icons/arrow-down-right.svg new file mode 100644 index 0000000..e0ca232 --- /dev/null +++ b/static/icons/arrow-down-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down-up.svg b/static/icons/arrow-down-up.svg new file mode 100644 index 0000000..6bab77f --- /dev/null +++ b/static/icons/arrow-down-up.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-down.svg b/static/icons/arrow-down.svg new file mode 100644 index 0000000..317b68a --- /dev/null +++ b/static/icons/arrow-down.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-left-2.svg b/static/icons/arrow-left-2.svg new file mode 100644 index 0000000..09ecec9 --- /dev/null +++ b/static/icons/arrow-left-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-left-right.svg b/static/icons/arrow-left-right.svg new file mode 100644 index 0000000..3915efb --- /dev/null +++ b/static/icons/arrow-left-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-left.svg b/static/icons/arrow-left.svg new file mode 100644 index 0000000..eb1b95f --- /dev/null +++ b/static/icons/arrow-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-right-2.svg b/static/icons/arrow-right-2.svg new file mode 100644 index 0000000..5647c5b --- /dev/null +++ b/static/icons/arrow-right-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-right-left.svg b/static/icons/arrow-right-left.svg new file mode 100644 index 0000000..b3dbbc2 --- /dev/null +++ b/static/icons/arrow-right-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-right.svg b/static/icons/arrow-right.svg new file mode 100644 index 0000000..98fd9e8 --- /dev/null +++ b/static/icons/arrow-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up-2.svg b/static/icons/arrow-up-2.svg new file mode 100644 index 0000000..ef31915 --- /dev/null +++ b/static/icons/arrow-up-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up-down.svg b/static/icons/arrow-up-down.svg new file mode 100644 index 0000000..e83a86f --- /dev/null +++ b/static/icons/arrow-up-down.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up-left-2.svg b/static/icons/arrow-up-left-2.svg new file mode 100644 index 0000000..e869fba --- /dev/null +++ b/static/icons/arrow-up-left-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up-left.svg b/static/icons/arrow-up-left.svg new file mode 100644 index 0000000..820469c --- /dev/null +++ b/static/icons/arrow-up-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up-right-2.svg b/static/icons/arrow-up-right-2.svg new file mode 100644 index 0000000..5e4bff7 --- /dev/null +++ b/static/icons/arrow-up-right-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up-right.svg b/static/icons/arrow-up-right.svg new file mode 100644 index 0000000..6c3ee2e --- /dev/null +++ b/static/icons/arrow-up-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrow-up.svg b/static/icons/arrow-up.svg new file mode 100644 index 0000000..c6e18b7 --- /dev/null +++ b/static/icons/arrow-up.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/arrows-diagonal-out.svg b/static/icons/arrows-diagonal-out.svg new file mode 100644 index 0000000..4840df3 --- /dev/null +++ b/static/icons/arrows-diagonal-out.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/average.svg b/static/icons/average.svg new file mode 100644 index 0000000..b427612 --- /dev/null +++ b/static/icons/average.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/bag-2.svg b/static/icons/bag-2.svg new file mode 100644 index 0000000..18e8261 --- /dev/null +++ b/static/icons/bag-2.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/bag-4.svg b/static/icons/bag-4.svg new file mode 100644 index 0000000..2614b52 --- /dev/null +++ b/static/icons/bag-4.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/bag-lock.svg b/static/icons/bag-lock.svg new file mode 100644 index 0000000..f7e9ece --- /dev/null +++ b/static/icons/bag-lock.svg @@ -0,0 +1,11 @@ + + + + + + \ No newline at end of file diff --git a/static/icons/bag-plus.svg b/static/icons/bag-plus.svg new file mode 100644 index 0000000..6bb5934 --- /dev/null +++ b/static/icons/bag-plus.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/bag-work.svg b/static/icons/bag-work.svg new file mode 100644 index 0000000..51247c2 --- /dev/null +++ b/static/icons/bag-work.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/bag.svg b/static/icons/bag.svg new file mode 100644 index 0000000..92d74b7 --- /dev/null +++ b/static/icons/bag.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/bolt.svg b/static/icons/bolt.svg new file mode 100644 index 0000000..fc64532 --- /dev/null +++ b/static/icons/bolt.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/bookmark.svg b/static/icons/bookmark.svg new file mode 100644 index 0000000..813568e --- /dev/null +++ b/static/icons/bookmark.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/burger-menu.svg b/static/icons/burger-menu.svg new file mode 100644 index 0000000..3b6b197 --- /dev/null +++ b/static/icons/burger-menu.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/calendar-2.svg b/static/icons/calendar-2.svg new file mode 100644 index 0000000..83fdf9f --- /dev/null +++ b/static/icons/calendar-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/calendar.svg b/static/icons/calendar.svg new file mode 100644 index 0000000..ec49336 --- /dev/null +++ b/static/icons/calendar.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/check-circle.svg b/static/icons/check-circle.svg new file mode 100644 index 0000000..12c6187 --- /dev/null +++ b/static/icons/check-circle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/check-square.svg b/static/icons/check-square.svg new file mode 100644 index 0000000..6208802 --- /dev/null +++ b/static/icons/check-square.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/check.svg b/static/icons/check.svg new file mode 100644 index 0000000..71b981c --- /dev/null +++ b/static/icons/check.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevron-down-circle.svg b/static/icons/chevron-down-circle.svg new file mode 100644 index 0000000..6eb3fe5 --- /dev/null +++ b/static/icons/chevron-down-circle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/chevron-down.svg b/static/icons/chevron-down.svg new file mode 100644 index 0000000..72235ec --- /dev/null +++ b/static/icons/chevron-down.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevron-left-circle.svg b/static/icons/chevron-left-circle.svg new file mode 100644 index 0000000..87719e4 --- /dev/null +++ b/static/icons/chevron-left-circle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/chevron-left.svg b/static/icons/chevron-left.svg new file mode 100644 index 0000000..774ee4b --- /dev/null +++ b/static/icons/chevron-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevron-right-circle.svg b/static/icons/chevron-right-circle.svg new file mode 100644 index 0000000..98d5421 --- /dev/null +++ b/static/icons/chevron-right-circle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/chevron-right.svg b/static/icons/chevron-right.svg new file mode 100644 index 0000000..19d783a --- /dev/null +++ b/static/icons/chevron-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevron-up-circle.svg b/static/icons/chevron-up-circle.svg new file mode 100644 index 0000000..750f239 --- /dev/null +++ b/static/icons/chevron-up-circle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/chevron-up.svg b/static/icons/chevron-up.svg new file mode 100644 index 0000000..6257f70 --- /dev/null +++ b/static/icons/chevron-up.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-down.svg b/static/icons/chevrons-down.svg new file mode 100644 index 0000000..8c7a4ae --- /dev/null +++ b/static/icons/chevrons-down.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-left.svg b/static/icons/chevrons-left.svg new file mode 100644 index 0000000..45ebf78 --- /dev/null +++ b/static/icons/chevrons-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-right.svg b/static/icons/chevrons-right.svg new file mode 100644 index 0000000..2b3aea1 --- /dev/null +++ b/static/icons/chevrons-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-triple-down.svg b/static/icons/chevrons-triple-down.svg new file mode 100644 index 0000000..ed4dc03 --- /dev/null +++ b/static/icons/chevrons-triple-down.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-triple-left.svg b/static/icons/chevrons-triple-left.svg new file mode 100644 index 0000000..3df85fe --- /dev/null +++ b/static/icons/chevrons-triple-left.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-triple-right.svg b/static/icons/chevrons-triple-right.svg new file mode 100644 index 0000000..aac50bf --- /dev/null +++ b/static/icons/chevrons-triple-right.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-triple-up.svg b/static/icons/chevrons-triple-up.svg new file mode 100644 index 0000000..0ef890b --- /dev/null +++ b/static/icons/chevrons-triple-up.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/chevrons-up.svg b/static/icons/chevrons-up.svg new file mode 100644 index 0000000..867f61b --- /dev/null +++ b/static/icons/chevrons-up.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/circle-dot.svg b/static/icons/circle-dot.svg new file mode 100644 index 0000000..e380e09 --- /dev/null +++ b/static/icons/circle-dot.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/circle-dots.svg b/static/icons/circle-dots.svg new file mode 100644 index 0000000..6275a98 --- /dev/null +++ b/static/icons/circle-dots.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/circle-slash.svg b/static/icons/circle-slash.svg new file mode 100644 index 0000000..861352d --- /dev/null +++ b/static/icons/circle-slash.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/circle.svg b/static/icons/circle.svg new file mode 100644 index 0000000..84f078d --- /dev/null +++ b/static/icons/circle.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/clipboard-alert.svg b/static/icons/clipboard-alert.svg new file mode 100644 index 0000000..3cfae40 --- /dev/null +++ b/static/icons/clipboard-alert.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/clipboard-chart.svg b/static/icons/clipboard-chart.svg new file mode 100644 index 0000000..9805ca0 --- /dev/null +++ b/static/icons/clipboard-chart.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/clipboard-check.svg b/static/icons/clipboard-check.svg new file mode 100644 index 0000000..4515c5e --- /dev/null +++ b/static/icons/clipboard-check.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/clipboard-copy.svg b/static/icons/clipboard-copy.svg new file mode 100644 index 0000000..d59fc74 --- /dev/null +++ b/static/icons/clipboard-copy.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/clipboard-data.svg b/static/icons/clipboard-data.svg new file mode 100644 index 0000000..aebd8f5 --- /dev/null +++ b/static/icons/clipboard-data.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/clipboard-download.svg b/static/icons/clipboard-download.svg new file mode 100644 index 0000000..bcb349e --- /dev/null +++ b/static/icons/clipboard-download.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/clipboard-line-chart.svg b/static/icons/clipboard-line-chart.svg new file mode 100644 index 0000000..a9d28a7 --- /dev/null +++ b/static/icons/clipboard-line-chart.svg @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/static/icons/clipboard-upload.svg b/static/icons/clipboard-upload.svg new file mode 100644 index 0000000..434b4b8 --- /dev/null +++ b/static/icons/clipboard-upload.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/clock-2.svg b/static/icons/clock-2.svg new file mode 100644 index 0000000..2b15f8b --- /dev/null +++ b/static/icons/clock-2.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/clock.svg b/static/icons/clock.svg new file mode 100644 index 0000000..88df7eb --- /dev/null +++ b/static/icons/clock.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/cloud-download.svg b/static/icons/cloud-download.svg new file mode 100644 index 0000000..f284c8f --- /dev/null +++ b/static/icons/cloud-download.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/cloud-upload.svg b/static/icons/cloud-upload.svg new file mode 100644 index 0000000..2a22e27 --- /dev/null +++ b/static/icons/cloud-upload.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/cloud.svg b/static/icons/cloud.svg new file mode 100644 index 0000000..a84e416 --- /dev/null +++ b/static/icons/cloud.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/code-2.svg b/static/icons/code-2.svg new file mode 100644 index 0000000..dbb6f8e --- /dev/null +++ b/static/icons/code-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/code.svg b/static/icons/code.svg new file mode 100644 index 0000000..47d6c90 --- /dev/null +++ b/static/icons/code.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/command-line.svg b/static/icons/command-line.svg new file mode 100644 index 0000000..d7c4a03 --- /dev/null +++ b/static/icons/command-line.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/content.svg b/static/icons/content.svg new file mode 100644 index 0000000..21105d0 --- /dev/null +++ b/static/icons/content.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/copy.svg b/static/icons/copy.svg new file mode 100644 index 0000000..867f6cc --- /dev/null +++ b/static/icons/copy.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/credit-card.svg b/static/icons/credit-card.svg new file mode 100644 index 0000000..758948d --- /dev/null +++ b/static/icons/credit-card.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/database.svg b/static/icons/database.svg new file mode 100644 index 0000000..dd2463d --- /dev/null +++ b/static/icons/database.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/dialog.svg b/static/icons/dialog.svg new file mode 100644 index 0000000..9d761b8 --- /dev/null +++ b/static/icons/dialog.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/document-edit-2.svg b/static/icons/document-edit-2.svg new file mode 100644 index 0000000..a4e567b --- /dev/null +++ b/static/icons/document-edit-2.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/document-edit.svg b/static/icons/document-edit.svg new file mode 100644 index 0000000..8f685c8 --- /dev/null +++ b/static/icons/document-edit.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/document-minus.svg b/static/icons/document-minus.svg new file mode 100644 index 0000000..94dd4d8 --- /dev/null +++ b/static/icons/document-minus.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/document-shield.svg b/static/icons/document-shield.svg new file mode 100644 index 0000000..c5732d4 --- /dev/null +++ b/static/icons/document-shield.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/dots-horizontal.svg b/static/icons/dots-horizontal.svg new file mode 100644 index 0000000..f8b9c23 --- /dev/null +++ b/static/icons/dots-horizontal.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/download-2.svg b/static/icons/download-2.svg new file mode 100644 index 0000000..2884115 --- /dev/null +++ b/static/icons/download-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/download-3.svg b/static/icons/download-3.svg new file mode 100644 index 0000000..f496817 --- /dev/null +++ b/static/icons/download-3.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/download.svg b/static/icons/download.svg new file mode 100644 index 0000000..595f3a5 --- /dev/null +++ b/static/icons/download.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/drop.svg b/static/icons/drop.svg new file mode 100644 index 0000000..eb0a474 --- /dev/null +++ b/static/icons/drop.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/east.svg b/static/icons/east.svg new file mode 100644 index 0000000..6082fd9 --- /dev/null +++ b/static/icons/east.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/equals.svg b/static/icons/equals.svg new file mode 100644 index 0000000..08896d9 --- /dev/null +++ b/static/icons/equals.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/eye.svg b/static/icons/eye.svg new file mode 100644 index 0000000..2f1f62f --- /dev/null +++ b/static/icons/eye.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-alert.svg b/static/icons/file-alert.svg new file mode 100644 index 0000000..bf5105b --- /dev/null +++ b/static/icons/file-alert.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-archive.svg b/static/icons/file-archive.svg new file mode 100644 index 0000000..1f76ed0 --- /dev/null +++ b/static/icons/file-archive.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-broken.svg b/static/icons/file-broken.svg new file mode 100644 index 0000000..2f9804f --- /dev/null +++ b/static/icons/file-broken.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-certificate-2.svg b/static/icons/file-certificate-2.svg new file mode 100644 index 0000000..bfc06f5 --- /dev/null +++ b/static/icons/file-certificate-2.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-certificate.svg b/static/icons/file-certificate.svg new file mode 100644 index 0000000..429da44 --- /dev/null +++ b/static/icons/file-certificate.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-chart.svg b/static/icons/file-chart.svg new file mode 100644 index 0000000..20a4a70 --- /dev/null +++ b/static/icons/file-chart.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-copy.svg b/static/icons/file-copy.svg new file mode 100644 index 0000000..d7d469e --- /dev/null +++ b/static/icons/file-copy.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-download.svg b/static/icons/file-download.svg new file mode 100644 index 0000000..36e1e61 --- /dev/null +++ b/static/icons/file-download.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-key.svg b/static/icons/file-key.svg new file mode 100644 index 0000000..c3e679a --- /dev/null +++ b/static/icons/file-key.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-lock.svg b/static/icons/file-lock.svg new file mode 100644 index 0000000..c4e5965 --- /dev/null +++ b/static/icons/file-lock.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-program.svg b/static/icons/file-program.svg new file mode 100644 index 0000000..a9ed60f --- /dev/null +++ b/static/icons/file-program.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-search.svg b/static/icons/file-search.svg new file mode 100644 index 0000000..357706c --- /dev/null +++ b/static/icons/file-search.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-signed.svg b/static/icons/file-signed.svg new file mode 100644 index 0000000..f3502c6 --- /dev/null +++ b/static/icons/file-signed.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-text.svg b/static/icons/file-text.svg new file mode 100644 index 0000000..ce116ff --- /dev/null +++ b/static/icons/file-text.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file-x.svg b/static/icons/file-x.svg new file mode 100644 index 0000000..6335c6f --- /dev/null +++ b/static/icons/file-x.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/file.svg b/static/icons/file.svg new file mode 100644 index 0000000..0b7d0d5 --- /dev/null +++ b/static/icons/file.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/filter-asc-2.svg b/static/icons/filter-asc-2.svg new file mode 100644 index 0000000..8ba8e96 --- /dev/null +++ b/static/icons/filter-asc-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/filter-asc-3.svg b/static/icons/filter-asc-3.svg new file mode 100644 index 0000000..58add88 --- /dev/null +++ b/static/icons/filter-asc-3.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/filter-asc.svg b/static/icons/filter-asc.svg new file mode 100644 index 0000000..1561654 --- /dev/null +++ b/static/icons/filter-asc.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/filter-desc-2.svg b/static/icons/filter-desc-2.svg new file mode 100644 index 0000000..de1f326 --- /dev/null +++ b/static/icons/filter-desc-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/filter-desc-3.svg b/static/icons/filter-desc-3.svg new file mode 100644 index 0000000..01c08ac --- /dev/null +++ b/static/icons/filter-desc-3.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/filter-desc.svg b/static/icons/filter-desc.svg new file mode 100644 index 0000000..77ddd9e --- /dev/null +++ b/static/icons/filter-desc.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/flag-2.svg b/static/icons/flag-2.svg new file mode 100644 index 0000000..34aca99 --- /dev/null +++ b/static/icons/flag-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/flag-3.svg b/static/icons/flag-3.svg new file mode 100644 index 0000000..ef18dc3 --- /dev/null +++ b/static/icons/flag-3.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/flag.svg b/static/icons/flag.svg new file mode 100644 index 0000000..d8a4bf9 --- /dev/null +++ b/static/icons/flag.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/folder.svg b/static/icons/folder.svg new file mode 100644 index 0000000..3a69c2e --- /dev/null +++ b/static/icons/folder.svg @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/static/icons/forbidden.svg b/static/icons/forbidden.svg new file mode 100644 index 0000000..1e0da82 --- /dev/null +++ b/static/icons/forbidden.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/forward.svg b/static/icons/forward.svg new file mode 100644 index 0000000..ac0969b --- /dev/null +++ b/static/icons/forward.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/globe-2.svg b/static/icons/globe-2.svg new file mode 100644 index 0000000..08d9809 --- /dev/null +++ b/static/icons/globe-2.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/static/icons/globe.svg b/static/icons/globe.svg new file mode 100644 index 0000000..eb59d66 --- /dev/null +++ b/static/icons/globe.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/static/icons/heart-2.svg b/static/icons/heart-2.svg new file mode 100644 index 0000000..4ea1f6e --- /dev/null +++ b/static/icons/heart-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/heart.svg b/static/icons/heart.svg new file mode 100644 index 0000000..240d3fc --- /dev/null +++ b/static/icons/heart.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/home.svg b/static/icons/home.svg new file mode 100644 index 0000000..082fe2d --- /dev/null +++ b/static/icons/home.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/inbox.svg b/static/icons/inbox.svg new file mode 100644 index 0000000..bb70339 --- /dev/null +++ b/static/icons/inbox.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/key.svg b/static/icons/key.svg new file mode 100644 index 0000000..1e13032 --- /dev/null +++ b/static/icons/key.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/link.svg b/static/icons/link.svg new file mode 100644 index 0000000..42d6be5 --- /dev/null +++ b/static/icons/link.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/location-pin.svg b/static/icons/location-pin.svg new file mode 100644 index 0000000..4462ab6 --- /dev/null +++ b/static/icons/location-pin.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/lock-open.svg b/static/icons/lock-open.svg new file mode 100644 index 0000000..b2a448e --- /dev/null +++ b/static/icons/lock-open.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/lock.svg b/static/icons/lock.svg new file mode 100644 index 0000000..dcd3d94 --- /dev/null +++ b/static/icons/lock.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/login.svg b/static/icons/login.svg new file mode 100644 index 0000000..fc30ad6 --- /dev/null +++ b/static/icons/login.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/logout.svg b/static/icons/logout.svg new file mode 100644 index 0000000..8b9e27d --- /dev/null +++ b/static/icons/logout.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/magnifier-2.svg b/static/icons/magnifier-2.svg new file mode 100644 index 0000000..e2c4593 --- /dev/null +++ b/static/icons/magnifier-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/magnifier-zoom-minus.svg b/static/icons/magnifier-zoom-minus.svg new file mode 100644 index 0000000..66e415b --- /dev/null +++ b/static/icons/magnifier-zoom-minus.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/magnifier-zoom-plus.svg b/static/icons/magnifier-zoom-plus.svg new file mode 100644 index 0000000..3be8d7d --- /dev/null +++ b/static/icons/magnifier-zoom-plus.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/magnifier.svg b/static/icons/magnifier.svg new file mode 100644 index 0000000..0b9507b --- /dev/null +++ b/static/icons/magnifier.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/mail.svg b/static/icons/mail.svg new file mode 100644 index 0000000..5508bb5 --- /dev/null +++ b/static/icons/mail.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/map.svg b/static/icons/map.svg new file mode 100644 index 0000000..31c5fd8 --- /dev/null +++ b/static/icons/map.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/message-check.svg b/static/icons/message-check.svg new file mode 100644 index 0000000..68bec13 --- /dev/null +++ b/static/icons/message-check.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/message-info.svg b/static/icons/message-info.svg new file mode 100644 index 0000000..66db685 --- /dev/null +++ b/static/icons/message-info.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/message.svg b/static/icons/message.svg new file mode 100644 index 0000000..d53bd41 --- /dev/null +++ b/static/icons/message.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/minus.svg b/static/icons/minus.svg new file mode 100644 index 0000000..43054e2 --- /dev/null +++ b/static/icons/minus.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/monitor.svg b/static/icons/monitor.svg new file mode 100644 index 0000000..6745a74 --- /dev/null +++ b/static/icons/monitor.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/moon-star.svg b/static/icons/moon-star.svg new file mode 100644 index 0000000..e5accb4 --- /dev/null +++ b/static/icons/moon-star.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/moon.svg b/static/icons/moon.svg new file mode 100644 index 0000000..7b3ad0b --- /dev/null +++ b/static/icons/moon.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/mouse-pointer.svg b/static/icons/mouse-pointer.svg new file mode 100644 index 0000000..5bd051f --- /dev/null +++ b/static/icons/mouse-pointer.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/new-tab-2.svg b/static/icons/new-tab-2.svg new file mode 100644 index 0000000..1529217 --- /dev/null +++ b/static/icons/new-tab-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/new-tab.svg b/static/icons/new-tab.svg new file mode 100644 index 0000000..64d63cb --- /dev/null +++ b/static/icons/new-tab.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/north.svg b/static/icons/north.svg new file mode 100644 index 0000000..dd9ac10 --- /dev/null +++ b/static/icons/north.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/notification.svg b/static/icons/notification.svg new file mode 100644 index 0000000..219835c --- /dev/null +++ b/static/icons/notification.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/pen-writing-3.svg b/static/icons/pen-writing-3.svg new file mode 100644 index 0000000..8fcba73 --- /dev/null +++ b/static/icons/pen-writing-3.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/pen-writing.svg b/static/icons/pen-writing.svg new file mode 100644 index 0000000..7bc2eac --- /dev/null +++ b/static/icons/pen-writing.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/pen.svg b/static/icons/pen.svg new file mode 100644 index 0000000..991bfd7 --- /dev/null +++ b/static/icons/pen.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/pin-flag.svg b/static/icons/pin-flag.svg new file mode 100644 index 0000000..52586e4 --- /dev/null +++ b/static/icons/pin-flag.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/pin-goal.svg b/static/icons/pin-goal.svg new file mode 100644 index 0000000..24222c3 --- /dev/null +++ b/static/icons/pin-goal.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/play.svg b/static/icons/play.svg new file mode 100644 index 0000000..9cb22e3 --- /dev/null +++ b/static/icons/play.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/plus.svg b/static/icons/plus.svg new file mode 100644 index 0000000..e665b91 --- /dev/null +++ b/static/icons/plus.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/pulse.svg b/static/icons/pulse.svg new file mode 100644 index 0000000..087e57a --- /dev/null +++ b/static/icons/pulse.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/reply.svg b/static/icons/reply.svg new file mode 100644 index 0000000..fdf9414 --- /dev/null +++ b/static/icons/reply.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/send-2.svg b/static/icons/send-2.svg new file mode 100644 index 0000000..bd05f41 --- /dev/null +++ b/static/icons/send-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/send.svg b/static/icons/send.svg new file mode 100644 index 0000000..1bb2f3a --- /dev/null +++ b/static/icons/send.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/server.svg b/static/icons/server.svg new file mode 100644 index 0000000..0a1c0a7 --- /dev/null +++ b/static/icons/server.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/settings-2.svg b/static/icons/settings-2.svg new file mode 100644 index 0000000..ffc0c11 --- /dev/null +++ b/static/icons/settings-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/settings-3.svg b/static/icons/settings-3.svg new file mode 100644 index 0000000..ce5be79 --- /dev/null +++ b/static/icons/settings-3.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/settings-4.svg b/static/icons/settings-4.svg new file mode 100644 index 0000000..3827d6a --- /dev/null +++ b/static/icons/settings-4.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/static/icons/settings.svg b/static/icons/settings.svg new file mode 100644 index 0000000..2015b2d --- /dev/null +++ b/static/icons/settings.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/share-2.svg b/static/icons/share-2.svg new file mode 100644 index 0000000..eabd15c --- /dev/null +++ b/static/icons/share-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/share.svg b/static/icons/share.svg new file mode 100644 index 0000000..6820704 --- /dev/null +++ b/static/icons/share.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/shield-disabled.svg b/static/icons/shield-disabled.svg new file mode 100644 index 0000000..5b25f59 --- /dev/null +++ b/static/icons/shield-disabled.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/static/icons/shield-key.svg b/static/icons/shield-key.svg new file mode 100644 index 0000000..8d27536 --- /dev/null +++ b/static/icons/shield-key.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/shield-lock.svg b/static/icons/shield-lock.svg new file mode 100644 index 0000000..bc4a33a --- /dev/null +++ b/static/icons/shield-lock.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/shield-user.svg b/static/icons/shield-user.svg new file mode 100644 index 0000000..ca73611 --- /dev/null +++ b/static/icons/shield-user.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/shield-x.svg b/static/icons/shield-x.svg new file mode 100644 index 0000000..d040044 --- /dev/null +++ b/static/icons/shield-x.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/shield.svg b/static/icons/shield.svg new file mode 100644 index 0000000..5d5621f --- /dev/null +++ b/static/icons/shield.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/south.svg b/static/icons/south.svg new file mode 100644 index 0000000..4238228 --- /dev/null +++ b/static/icons/south.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/square.svg b/static/icons/square.svg new file mode 100644 index 0000000..cc5e940 --- /dev/null +++ b/static/icons/square.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/sun.svg b/static/icons/sun.svg new file mode 100644 index 0000000..d292182 --- /dev/null +++ b/static/icons/sun.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/tag.svg b/static/icons/tag.svg new file mode 100644 index 0000000..1bd6bee --- /dev/null +++ b/static/icons/tag.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/terminal.svg b/static/icons/terminal.svg new file mode 100644 index 0000000..67d48be --- /dev/null +++ b/static/icons/terminal.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/test-tube.svg b/static/icons/test-tube.svg new file mode 100644 index 0000000..3b8ce6b --- /dev/null +++ b/static/icons/test-tube.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/translate.svg b/static/icons/translate.svg new file mode 100644 index 0000000..76eebf5 --- /dev/null +++ b/static/icons/translate.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/trash-2.svg b/static/icons/trash-2.svg new file mode 100644 index 0000000..58e9b72 --- /dev/null +++ b/static/icons/trash-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/trash-alert.svg b/static/icons/trash-alert.svg new file mode 100644 index 0000000..e56eca9 --- /dev/null +++ b/static/icons/trash-alert.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/trash.svg b/static/icons/trash.svg new file mode 100644 index 0000000..c236be7 --- /dev/null +++ b/static/icons/trash.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/trend-down.svg b/static/icons/trend-down.svg new file mode 100644 index 0000000..eeb26c6 --- /dev/null +++ b/static/icons/trend-down.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/trend-flatish.svg b/static/icons/trend-flatish.svg new file mode 100644 index 0000000..39c6c30 --- /dev/null +++ b/static/icons/trend-flatish.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/trend-up.svg b/static/icons/trend-up.svg new file mode 100644 index 0000000..9b90cac --- /dev/null +++ b/static/icons/trend-up.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/upload-2.svg b/static/icons/upload-2.svg new file mode 100644 index 0000000..eed5c4a --- /dev/null +++ b/static/icons/upload-2.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/upload.svg b/static/icons/upload.svg new file mode 100644 index 0000000..09e7ce4 --- /dev/null +++ b/static/icons/upload.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/user-circle.svg b/static/icons/user-circle.svg new file mode 100644 index 0000000..9f4bb63 --- /dev/null +++ b/static/icons/user-circle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/static/icons/user.svg b/static/icons/user.svg new file mode 100644 index 0000000..5cdf24e --- /dev/null +++ b/static/icons/user.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/video-camera.svg b/static/icons/video-camera.svg new file mode 100644 index 0000000..0d2bd30 --- /dev/null +++ b/static/icons/video-camera.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/west.svg b/static/icons/west.svg new file mode 100644 index 0000000..8b5ea0b --- /dev/null +++ b/static/icons/west.svg @@ -0,0 +1,3 @@ + + + diff --git a/static/icons/x.svg b/static/icons/x.svg new file mode 100644 index 0000000..b209221 --- /dev/null +++ b/static/icons/x.svg @@ -0,0 +1,3 @@ + + + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..a5de3db --- /dev/null +++ b/templates/index.html @@ -0,0 +1,718 @@ + + + + + ngxstat Reports + + + + +
+

ngxstat Reports

+ + + +
+ + + + + + + + + +
+ +
+
+ +
+
+

Recent

+

Total logs: -

+

Date range: - to -

+

Unique domains: -

+

Last generated: -

+

Generation time: - seconds

+
+ +
+
+
+ + + + + + +
+ + + + + + diff --git a/templates/report.html b/templates/report.html deleted file mode 100644 index e6dfee0..0000000 --- a/templates/report.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - {{ interval.title() }} Report - - - - -
-

{{ interval.title() }} Report

- -
- - - diff --git a/templates/report_snippet.html b/templates/report_snippet.html new file mode 100644 index 0000000..2c0ff62 --- /dev/null +++ b/templates/report_snippet.html @@ -0,0 +1,11 @@ +
+

+ {% if report.icon %}{{ report.icon }}{% endif %} + {{ report.label }} +

+ {% if report.chart == 'table' %} +
+ {% else %} + + {% endif %} +
diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 0000000..6e97ab6 --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,318 @@ +import json +import sqlite3 +from pathlib import Path +from scripts import analyze +from scripts import generate_reports as gr + + +def setup_db(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ip TEXT, + host TEXT, + time TEXT, + request TEXT, + status INTEGER, + bytes_sent INTEGER, + referer TEXT, + user_agent TEXT, + cache_status TEXT + ) + """ + ) + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "127.0.0.1", + "example.com", + "2024-01-01 10:00:00", + "GET / HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ) + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "127.0.0.1", + "missing.com", + "2024-01-01 11:00:00", + "GET / HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ) + conn.commit() + conn.close() + + +def test_check_missing_domains(tmp_path, monkeypatch, capsys): + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + + conf = tmp_path / "nginx.conf" + conf.write_text( + """ +server { + listen 80; + server_name example.com; +} +""" + ) + + monkeypatch.setattr(analyze, "DB_PATH", db_path) + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(analyze.nginx_config, "DEFAULT_PATHS", [str(conf)]) + + analyze.check_missing_domains(json_output=False) + out = capsys.readouterr().out.strip().splitlines() + assert out == ["missing.com"] + + analyze.check_missing_domains(json_output=True) + out_json = json.loads(capsys.readouterr().out.strip()) + assert out_json == ["missing.com"] + + +def test_suggest_cache(tmp_path, monkeypatch, capsys): + db_path = tmp_path / "database" / "ngxstat.db" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ip TEXT, + host TEXT, + time TEXT, + request TEXT, + status INTEGER, + bytes_sent INTEGER, + referer TEXT, + user_agent TEXT, + cache_status TEXT + ) + """ + ) + entries = [ + ( + "127.0.0.1", + "example.com", + "2024-01-01 10:00:00", + "GET /foo HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ( + "127.0.0.1", + "example.com", + "2024-01-01 10:01:00", + "GET /foo HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ( + "127.0.0.1", + "example.com", + "2024-01-01 10:02:00", + "GET /foo HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ( + "127.0.0.1", + "cached.com", + "2024-01-01 10:00:00", + "GET /bar HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ( + "127.0.0.1", + "cached.com", + "2024-01-01 10:01:00", + "GET /bar HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ] + cur.executemany( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + entries, + ) + conn.commit() + conn.close() + + conf = tmp_path / "nginx.conf" + conf.write_text( + """ +server { + listen 80; + server_name example.com; +} + +server { + listen 80; + server_name cached.com; + proxy_cache cache1; +} +""" + ) + + monkeypatch.setattr(analyze, "DB_PATH", db_path) + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(analyze.nginx_config, "DEFAULT_PATHS", [str(conf)]) + + analyze.suggest_cache(threshold=2, json_output=False) + out = capsys.readouterr().out.strip().splitlines() + assert out == ["example.com /foo 3"] + + analyze.suggest_cache(threshold=2, json_output=True) + out_json = json.loads(capsys.readouterr().out.strip()) + assert out_json == [{"host": "example.com", "path": "/foo", "misses": 3}] + + +def setup_threat_db(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ip TEXT, + host TEXT, + time TEXT, + request TEXT, + status INTEGER, + bytes_sent INTEGER, + referer TEXT, + user_agent TEXT, + cache_status TEXT + ) + """ + ) + + # Previous hour traffic with no errors + for i in range(10): + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "2.2.2.2", + "example.com", + f"2024-01-01 11:{i:02d}:00", + "GET /ok HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ) + + # Recent hour with errors + for i in range(10): + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "3.3.3.3", + "example.com", + f"2024-01-01 12:{i:02d}:00", + "GET /fail HTTP/1.1", + 500, + 100, + "-", + "curl", + "MISS", + ), + ) + + # High traffic from single IP + for i in range(101): + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "1.1.1.1", + "example.net", + f"2024-01-01 12:{i % 10:02d}:30", + "GET /spam HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ) + + # New suspicious user agent + for i in range(15): + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "4.4.4.4", + "example.org", + f"2024-01-01 12:{30 + i:02d}:45", + "GET /bot HTTP/1.1", + 200, + 100, + "-", + "newbot", + "MISS", + ), + ) + + conn.commit() + conn.close() + + +def test_detect_threats(tmp_path, monkeypatch): + db_path = tmp_path / "database" / "ngxstat.db" + setup_threat_db(db_path) + + out_dir = tmp_path / "analysis" + monkeypatch.setattr(analyze, "DB_PATH", db_path) + monkeypatch.setattr(analyze, "ANALYSIS_DIR", out_dir) + + analyze.detect_threats(hours=1, ip_threshold=100) + + report = json.loads((out_dir / "threat_report.json").read_text()) + + hosts = {e["host"] for e in report.get("error_spikes", [])} + assert "example.com" in hosts + + ips = {e["ip"] for e in report.get("high_ip_requests", [])} + assert "1.1.1.1" in ips + + agents = {e["user_agent"] for e in report.get("suspicious_agents", [])} + assert "newbot" in agents diff --git a/tests/test_importer.py b/tests/test_importer.py index 349786b..38c9be9 100644 --- a/tests/test_importer.py +++ b/tests/test_importer.py @@ -15,10 +15,10 @@ def sample_logs(tmp_path): log_dir.mkdir(parents=True, exist_ok=True) (log_dir / "access.log.1").write_text( - "127.0.0.1 - example.com [01/Jan/2024:10:00:00 +0000] \"GET / HTTP/1.1\" 200 123 \"-\" \"curl\" MISS\n" + '127.0.0.1 - example.com [01/Jan/2024:10:00:00 +0000] "GET / HTTP/1.1" 200 123 "-" "curl" MISS\n' ) (log_dir / "access.log").write_text( - "127.0.0.1 - example.com [01/Jan/2024:10:05:00 +0000] \"GET /about HTTP/1.1\" 200 123 \"-\" \"curl\" MISS\n" + '127.0.0.1 - example.com [01/Jan/2024:10:05:00 +0000] "GET /about HTTP/1.1" 200 123 "-" "curl" MISS\n' ) yield log_dir @@ -59,4 +59,3 @@ def test_idempotent_import(sample_logs, tmp_path): assert first_count == 2 assert second_count == first_count - diff --git a/tests/test_nginx_config.py b/tests/test_nginx_config.py new file mode 100644 index 0000000..cba4212 --- /dev/null +++ b/tests/test_nginx_config.py @@ -0,0 +1,63 @@ +from scripts import nginx_config as nc + + +def test_discover_configs(tmp_path, monkeypatch): + root = tmp_path / "nginx" + root.mkdir() + conf_d = root / "conf.d" + conf_d.mkdir() + subdir = root / "sub" + subdir.mkdir() + + main = root / "nginx.conf" + site = conf_d / "site.conf" + extra = root / "extra.conf" + nested = subdir / "foo.conf" + + main.write_text("include conf.d/*.conf;\ninclude extra.conf;\n") + site.write_text("# site config\n") + extra.write_text("include sub/foo.conf;\n") + nested.write_text("# nested config\n") + + monkeypatch.setattr(nc, "DEFAULT_PATHS", [str(main)]) + found = nc.discover_configs() + + assert found == {main, site, extra, nested} + + +def test_parse_servers(tmp_path): + conf1 = tmp_path / "site.conf" + conf2 = tmp_path / "other.conf" + + conf1.write_text( + """ +server { + listen 80; + server_name example.com; + root /srv/example; + proxy_cache cache1; +} +""" + ) + + conf2.write_text( + """ +server { + listen 443 ssl; + server_name example.org; +} +""" + ) + + servers = nc.parse_servers({conf1, conf2, tmp_path / "missing.conf"}) + servers = sorted(servers, key=lambda s: s.get("server_name")) + + assert len(servers) == 2 + assert servers[0]["server_name"] == "example.com" + assert servers[0]["listen"] == "80" + assert servers[0]["root"] == "/srv/example" + assert servers[0]["proxy_cache"] == "cache1" + + assert servers[1]["server_name"] == "example.org" + assert servers[1]["listen"] == "443 ssl" + assert "proxy_cache" not in servers[1] diff --git a/tests/test_reports.py b/tests/test_reports.py new file mode 100644 index 0000000..60a6df6 --- /dev/null +++ b/tests/test_reports.py @@ -0,0 +1,415 @@ +import sqlite3 +from pathlib import Path +import json +from datetime import datetime + +import pytest +from typer.testing import CliRunner +from scripts import generate_reports as gr + + +def setup_db(path: Path): + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE logs ( + id INTEGER PRIMARY KEY, + ip TEXT, + host TEXT, + time TEXT, + request TEXT, + status INTEGER, + bytes_sent INTEGER, + referer TEXT, + user_agent TEXT, + cache_status TEXT + ) + """ + ) + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "127.0.0.1", + "example.com", + "2024-01-01 10:00:00", + "GET / HTTP/1.1", + 200, + 100, + "-", + "curl", + "MISS", + ), + ) + cur.execute( + "INSERT INTO logs (ip, host, time, request, status, bytes_sent, referer, user_agent, cache_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "127.0.0.1", + "example.com", + "2024-01-01 10:05:00", + "GET /err HTTP/1.1", + 500, + 100, + "-", + "curl", + "MISS", + ), + ) + conn.commit() + conn.close() + + +@pytest.fixture() +def sample_reports(tmp_path): + cfg = tmp_path / "reports.yml" + cfg.write_text( + """ +- name: hits + query: | + SELECT {bucket} AS bucket, COUNT(*) AS value + FROM logs + GROUP BY bucket + ORDER BY bucket +- name: error_rate + query: | + SELECT {bucket} AS bucket, + SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) * 100.0 / COUNT(*) AS value + FROM logs + GROUP BY bucket + ORDER BY bucket +- name: domain_traffic + per_domain: false + query: | + SELECT host AS bucket, + COUNT(*) AS value + FROM logs + GROUP BY host + ORDER BY value DESC +- name: skip_report + per_domain: false + query: | + SELECT {bucket} AS bucket, COUNT(*) AS value + FROM logs + GROUP BY bucket + ORDER BY bucket +- name: domain_totals + global: true + query: | + SELECT host AS bucket, + COUNT(*) AS value + FROM logs + GROUP BY host + ORDER BY value DESC +""" + ) + return cfg + + +def test_generate_interval(tmp_path, sample_reports, monkeypatch): + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output") + monkeypatch.setattr(gr, "REPORT_CONFIG", sample_reports) + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) + + gr._generate_interval("hourly") + + hits = json.loads((tmp_path / "output" / "hourly" / "hits.json").read_text()) + assert hits[0]["value"] == 2 + error_rate = json.loads( + (tmp_path / "output" / "hourly" / "error_rate.json").read_text() + ) + assert error_rate[0]["value"] == pytest.approx(50.0) + reports = json.loads((tmp_path / "output" / "hourly" / "reports.json").read_text()) + assert {r["name"] for r in reports} == {"hits", "error_rate", "skip_report"} + for r in reports: + snippet = tmp_path / "output" / "hourly" / r["html"] + assert snippet.exists() + + +def test_generate_interval_domain_filter(tmp_path, sample_reports, monkeypatch): + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output") + monkeypatch.setattr(gr, "REPORT_CONFIG", sample_reports) + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) + + gr._generate_interval("hourly", "example.com") + + hits = json.loads( + ( + tmp_path / "output" / "domains" / "example.com" / "hourly" / "hits.json" + ).read_text() + ) + assert hits[0]["value"] == 2 + reports = json.loads( + ( + tmp_path / "output" / "domains" / "example.com" / "hourly" / "reports.json" + ).read_text() + ) + assert {r["name"] for r in reports} == {"hits", "error_rate"} + assert not ( + tmp_path / "output" / "domains" / "example.com" / "hourly" / "skip_report.json" + ).exists() + + +def test_generate_root_index(tmp_path, sample_reports, monkeypatch): + db_path = tmp_path / "database" / "ngxstat.db" + setup_db(db_path) + + monkeypatch.setattr(gr, "DB_PATH", db_path) + monkeypatch.setattr(gr, "OUTPUT_DIR", tmp_path / "output") + monkeypatch.setattr(gr, "REPORT_CONFIG", sample_reports) + monkeypatch.setattr( + gr, "TEMPLATE_DIR", Path(__file__).resolve().parents[1] / "templates" + ) + + gr._generate_interval("hourly") + gr._generate_interval("daily") + + # create dummy domain directories + (tmp_path / "output" / "domains" / "foo.com").mkdir(parents=True) + (tmp_path / "output" / "domains" / "bar.com").mkdir(parents=True) + # add an extra directory with capitalized name to ensure it's ignored + (tmp_path / "output" / "Global").mkdir(parents=True) + # add an analysis directory to ensure it's excluded + (tmp_path / "output" / "analysis").mkdir(parents=True) + + gr._generate_root_index() + + index_file = tmp_path / "output" / "index.html" + assert index_file.exists() + content = index_file.read_text() + + # check for interval options + assert '