Add analysis tab and JSON outputs

README.md

@@ -98,6 +98,8 @@ threats.
 ```bash
 ./run-analysis.sh
 ```
+The JSON results are written under `output/analysis` and can be viewed from the
+"Analysis" tab in the generated dashboard.
 ## Serving Reports with Nginx
 
 To expose the generated HTML dashboards and JSON files over HTTP you can use a

scripts/analyze.py

@@ -138,6 +138,10 @@ def check_missing_domains(json_output: bool = typer.Option(False, "--json", help
 
     missing = sorted(db_domains - config_domains)
 
+    ANALYSIS_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = ANALYSIS_DIR / "missing_domains.json"
+    out_path.write_text(json.dumps(missing, indent=2))
+
     if json_output:
         typer.echo(json.dumps(missing))
     else:
@@ -189,14 +193,19 @@ def suggest_cache(
     rows = [r for r in cur.fetchall() if r[0] in no_cache]
     conn.close()
 
+    result = [
+        {"host": host, "path": path, "misses": count} for host, path, count in rows
+    ]
+
+    ANALYSIS_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = ANALYSIS_DIR / "cache_suggestions.json"
+    out_path.write_text(json.dumps(result, indent=2))
+
     if json_output:
-        result = [
-            {"host": host, "path": path, "misses": count} for host, path, count in rows
-        ]
         typer.echo(json.dumps(result))
     else:
-        for host, path, count in rows:
+        for item in result:
-            typer.echo(f"{host} {path} {count}")
+            typer.echo(f"{item['host']} {item['path']} {item['misses']}")
 
 
 @app.command("detect-threats")

templates/index.html

@@ -15,6 +15,7 @@
         <li class="is-active" data-tab="overview"><a>Overview</a></li>
         <li data-tab="all"><a>All Domains</a></li>
         <li data-tab="domain"><a>Per Domain</a></li>
+        <li data-tab="analysis"><a>Analysis</a></li>
       </ul>
     </div>
 
@@ -56,9 +57,15 @@
       <div id="reports-all"></div>
     </div>
 
-    <div id="domain-section" class="is-hidden">
+  <div id="domain-section" class="is-hidden">
-      <div id="reports-domain"></div>
+    <div id="reports-domain"></div>
-    </div>
+  </div>
+
+  <div id="analysis-section" class="is-hidden">
+    <div id="analysis-missing" class="box"></div>
+    <div id="analysis-cache" class="box mt-5"></div>
+    <div id="analysis-threats" class="box mt-5"></div>
+  </div>
   </div>
   <script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script>
   <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
@@ -73,13 +80,19 @@
     const sections = {
       overview: document.getElementById('overview-section'),
       all: document.getElementById('all-section'),
-      domain: document.getElementById('domain-section')
+      domain: document.getElementById('domain-section'),
+      analysis: document.getElementById('analysis-section')
     };
     const containers = {
       overview: document.getElementById('overview-reports'),
       all: document.getElementById('reports-all'),
       domain: document.getElementById('reports-domain')
     };
+    const analysisElems = {
+      missing: document.getElementById('analysis-missing'),
+      cache: document.getElementById('analysis-cache'),
+      threats: document.getElementById('analysis-threats')
+    };
     const totalElem = document.getElementById('stat-total');
     const startElem = document.getElementById('stat-start');
     const endElem = document.getElementById('stat-end');
@@ -169,19 +182,99 @@
         path = 'domains/' + encodeURIComponent(currentDomain) + '/' + currentInterval;
       }
 
-      fetch(path + '/reports.json')
+        fetch(path + '/reports.json')
-        .then(r => r.json())
+          .then(r => r.json())
-        .then(reports => {
+          .then(reports => {
-          container.innerHTML = '';
+            container.innerHTML = '';
-          reports.forEach(rep => {
+            reports.forEach(rep => {
-            fetch(path + '/' + rep.html)
+              fetch(path + '/' + rep.html)
-              .then(r => r.text())
+                .then(r => r.text())
-              .then(html => {
+                .then(html => {
-                container.insertAdjacentHTML('beforeend', html);
+                  container.insertAdjacentHTML('beforeend', html);
-                initReport(rep, path);
+                  initReport(rep, path);
-              });
+                });
+            });
+            feather.replace();
           });
-          feather.replace();
+    }
+
+    function loadAnalysis() {
+      analysisElems.missing.innerHTML = '<h2 class="subtitle">Missing Domains</h2>';
+      analysisElems.cache.innerHTML = '<h2 class="subtitle">Cache Suggestions</h2>';
+      analysisElems.threats.innerHTML = '<h2 class="subtitle">Threat Report</h2>';
+
+      fetch('analysis/missing_domains.json')
+        .then(r => r.json())
+        .then(list => {
+          if (list.length === 0) {
+            analysisElems.missing.insertAdjacentHTML('beforeend', '<p>None</p>');
+            return;
+          }
+          const items = list.map(d => `<li>${d}</li>`).join('');
+          analysisElems.missing.insertAdjacentHTML('beforeend', `<ul>${items}</ul>`);
+        });
+
+      fetch('analysis/cache_suggestions.json')
+        .then(r => r.json())
+        .then(data => {
+          if (data.length === 0) {
+            analysisElems.cache.insertAdjacentHTML('beforeend', '<p>No suggestions</p>');
+            return;
+          }
+          analysisElems.cache.insertAdjacentHTML('beforeend', '<table id="table-cache" class="table is-striped"></table>');
+          const rows = data.map(x => [x.host, x.path, x.misses]);
+          new DataTable('#table-cache', {
+            data: rows,
+            columns: [
+              { title: 'Domain' },
+              { title: 'Path' },
+              { title: 'Misses' }
+            ]
+          });
+        });
+
+      fetch('analysis/threat_report.json')
+        .then(r => r.json())
+        .then(rep => {
+          const hasData = rep.error_spikes?.length || rep.suspicious_agents?.length || rep.high_ip_requests?.length;
+          if (!hasData) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<p>No threats detected</p>');
+            return;
+          }
+          if (rep.error_spikes && rep.error_spikes.length) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<h3 class="subtitle is-6 mt-4">Error Spikes</h3><table id="table-errors" class="table is-striped"></table>');
+            const rows = rep.error_spikes.map(x => [x.host, x.recent_error_rate, x.previous_error_rate]);
+            new DataTable('#table-errors', {
+              data: rows,
+              columns: [
+                { title: 'Domain' },
+                { title: 'Recent %' },
+                { title: 'Previous %' }
+              ]
+            });
+          }
+          if (rep.suspicious_agents && rep.suspicious_agents.length) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<h3 class="subtitle is-6 mt-4">Suspicious User Agents</h3><table id="table-agents" class="table is-striped"></table>');
+            const rows = rep.suspicious_agents.map(x => [x.user_agent, x.requests]);
+            new DataTable('#table-agents', {
+              data: rows,
+              columns: [
+                { title: 'User Agent' },
+                { title: 'Requests' }
+              ]
+            });
+          }
+          if (rep.high_ip_requests && rep.high_ip_requests.length) {
+            analysisElems.threats.insertAdjacentHTML('beforeend', '<h3 class="subtitle is-6 mt-4">High IP Requests</h3><table id="table-ips" class="table is-striped"></table>');
+            const rows = rep.high_ip_requests.map(x => [x.ip, x.requests]);
+            new DataTable('#table-ips', {
+              data: rows,
+              columns: [
+                { title: 'IP' },
+                { title: 'Requests' }
+              ]
+            });
+          }
         });
     }
 
@@ -198,7 +291,11 @@
       if (name === 'overview') {
         loadStats();
       }
-      loadReports();
+      if (name === 'analysis') {
+        loadAnalysis();
+      } else {
+        loadReports();
+      }
     }
 
     intervalSelect.addEventListener('change', () => {