Merge pull request #37 from wagesj45/codex/add-analysis-tab-to-report-page

Add analysis tab with JSON output

README.md

@@ -98,6 +98,8 @@ threats.
 ```bash
 ./run-analysis.sh
 ```
+The JSON results are written under `output/analysis` and can be viewed from the
+"Analysis" tab in the generated dashboard.
 ## Serving Reports with Nginx
 
 To expose the generated HTML dashboards and JSON files over HTTP you can use a

scripts/analyze.py

@@ -138,6 +138,10 @@ def check_missing_domains(json_output: bool = typer.Option(False, "--json", help
 
     missing = sorted(db_domains - config_domains)
 
+    ANALYSIS_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = ANALYSIS_DIR / "missing_domains.json"
+    out_path.write_text(json.dumps(missing, indent=2))
+
     if json_output:
         typer.echo(json.dumps(missing))
     else:
@@ -189,14 +193,19 @@ def suggest_cache(
     rows = [r for r in cur.fetchall() if r[0] in no_cache]
     conn.close()
 
-    if json_output:
     result = [
         {"host": host, "path": path, "misses": count} for host, path, count in rows
     ]
+
+    ANALYSIS_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = ANALYSIS_DIR / "cache_suggestions.json"
+    out_path.write_text(json.dumps(result, indent=2))
+
+    if json_output:
         typer.echo(json.dumps(result))
     else:
-        for host, path, count in rows:
-            typer.echo(f"{host} {path} {count}")
+        for item in result:
+            typer.echo(f"{item['host']} {item['path']} {item['misses']}")
 
 
 @app.command("detect-threats")

templates/index.html

@@ -15,6 +15,7 @@
         <li class="is-active" data-tab="overview"><a>Overview</a></li>
         <li data-tab="all"><a>All Domains</a></li>
         <li data-tab="domain"><a>Per Domain</a></li>
+        <li data-tab="analysis"><a>Analysis</a></li>
       </ul>
     </div>
 
@@ -59,6 +60,12 @@
   <div id="domain-section" class="is-hidden">
     <div id="reports-domain"></div>
   </div>
+
+  <div id="analysis-section" class="is-hidden">
+    <div id="analysis-missing" class="box"></div>
+    <div id="analysis-cache" class="box mt-5"></div>
+    <div id="analysis-threats" class="box mt-5"></div>
+  </div>
   </div>
   <script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script>
   <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
@@ -73,13 +80,19 @@
     const sections = {
       overview: document.getElementById('overview-section'),
       all: document.getElementById('all-section'),
-      domain: document.getElementById('domain-section')
+      domain: document.getElementById('domain-section'),
+      analysis: document.getElementById('analysis-section')
     };
     const containers = {
       overview: document.getElementById('overview-reports'),
       all: document.getElementById('reports-all'),
       domain: document.getElementById('reports-domain')
     };
+    const analysisElems = {
+      missing: document.getElementById('analysis-missing'),
+      cache: document.getElementById('analysis-cache'),
+      threats: document.getElementById('analysis-threats')
+    };
     const totalElem = document.getElementById('stat-total');
     const startElem = document.getElementById('stat-start');
     const endElem = document.getElementById('stat-end');
@@ -185,6 +198,86 @@
           });
     }
 
+    function loadAnalysis() {
+      analysisElems.missing.innerHTML = '<h2 class="subtitle">Missing Domains</h2>';
+      analysisElems.cache.innerHTML = '<h2 class="subtitle">Cache Suggestions</h2>';
+      analysisElems.threats.innerHTML = '<h2 class="subtitle">Threat Report</h2>';
+
+      fetch('analysis/missing_domains.json')
+        .then(r => r.json())
+        .then(list => {
+          if (list.length === 0) {
+            analysisElems.missing.insertAdjacentHTML('beforeend', '<p>None</p>');
+            return;
+          }
+          const items = list.map(d => `<li>${d}</li>`).join('');
+          analysisElems.missing.insertAdjacentHTML('beforeend', `<ul>${items}</ul>`);
+        });
+
+      fetch('analysis/cache_suggestions.json')
+        .then(r => r.json())
+        .then(data => {
+          if (data.length === 0) {
+            analysisElems.cache.insertAdjacentHTML('beforeend', '<p>No suggestions</p>');
+            return;
+          }
+          analysisElems.cache.insertAdjacentHTML('beforeend', '<table id="table-cache" class="table is-striped"></table>');
+          const rows = data.map(x => [x.host, x.path, x.misses]);
+          new DataTable('#table-cache', {
+            data: rows,
+            columns: [
+              { title: 'Domain' },
+              { title: 'Path' },
+              { title: 'Misses' }
+            ]
+          });
+        });
+
+      fetch('analysis/threat_report.json')
+        .then(r => r.json())
+        .then(rep => {
+          const hasData = rep.error_spikes?.length || rep.suspicious_agents?.length || rep.high_ip_requests?.length;
+          if (!hasData) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<p>No threats detected</p>');
+            return;
+          }
+          if (rep.error_spikes && rep.error_spikes.length) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<h3 class="subtitle is-6 mt-4">Error Spikes</h3><table id="table-errors" class="table is-striped"></table>');
+            const rows = rep.error_spikes.map(x => [x.host, x.recent_error_rate, x.previous_error_rate]);
+            new DataTable('#table-errors', {
+              data: rows,
+              columns: [
+                { title: 'Domain' },
+                { title: 'Recent %' },
+                { title: 'Previous %' }
+              ]
+            });
+          }
+          if (rep.suspicious_agents && rep.suspicious_agents.length) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<h3 class="subtitle is-6 mt-4">Suspicious User Agents</h3><table id="table-agents" class="table is-striped"></table>');
+            const rows = rep.suspicious_agents.map(x => [x.user_agent, x.requests]);
+            new DataTable('#table-agents', {
+              data: rows,
+              columns: [
+                { title: 'User Agent' },
+                { title: 'Requests' }
+              ]
+            });
+          }
+          if (rep.high_ip_requests && rep.high_ip_requests.length) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<h3 class="subtitle is-6 mt-4">High IP Requests</h3><table id="table-ips" class="table is-striped"></table>');
+            const rows = rep.high_ip_requests.map(x => [x.ip, x.requests]);
+            new DataTable('#table-ips', {
+              data: rows,
+              columns: [
+                { title: 'IP' },
+                { title: 'Requests' }
+              ]
+            });
+          }
+        });
+    }
+
     function switchTab(name) {
       currentTab = name;
       tabs.forEach(tab => {
@@ -198,8 +291,12 @@
       if (name === 'overview') {
         loadStats();
       }
+      if (name === 'analysis') {
+        loadAnalysis();
+      } else {
         loadReports();
       }
+    }
 
     intervalSelect.addEventListener('change', () => {
       currentInterval = intervalSelect.value;