System performance metrics and rescoring statistics across {sessions.length} sessions.
++ Audit every autonomous rescoring decision. {filtered.length.toLocaleString()} records. +
+| Replaced | +|||||
|---|---|---|---|---|---|
| + {new Date(d.timestamp).toLocaleTimeString("en-US", { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + })} + | ++ {d.audio_file.replace(".mp3", "")} + | +{d.original_word} | ++ {d.action === "replaced" ? ( + {d.replacement_word} + ) : ( + (kept) + )} + | ++ + {d.whisper_confidence.toFixed(2)} + + | ++ {d.action === "replaced" ? ( + replaced + ) : ( + kept + )} + | +
|
+ |
+
+ Download audit logs, incident histories, and session aggregated metrics for + compliance reporting or offline analysis. +
++ Real-time visibility into every decision made by the shallow fusion + rescoring system. Monitor, audit, and validate autonomous transcription + corrections. +
+ + {/* Live Chart Preview */} ++ {description} +
+ + ); +} diff --git a/rescoring-dashboard/src/app/safety/page.tsx b/rescoring-dashboard/src/app/safety/page.tsx new file mode 100644 index 0000000..a8fa8d7 --- /dev/null +++ b/rescoring-dashboard/src/app/safety/page.tsx @@ -0,0 +1,282 @@ +"use client"; + +import { useState, useMemo } from "react"; +import { + generateDecisions, + generateIncidents, + generateSessions, + computeMetrics, + type Incident, +} from "../lib/mockData"; + +export default function AlertsPage() { + const decisions = useMemo(() => generateDecisions(200), []); + const incidents = useMemo(() => generateIncidents(), []); + const sessions = useMemo(() => generateSessions(), []); + const metrics = computeMetrics(decisions); + + const activeIncidents = incidents.filter((i) => !i.resolved); + const resolvedIncidents = incidents.filter((i) => i.resolved); + + const [dismissed, setDismissed] = useStateSafety monitoring, incident logs, and system health indicators.
+{incident.description}
+Generated: {datetime.now()}
" + html += export_df.to_html(classes='table table-striped', index=False) + html += "" + with open(path, 'w') as f: + f.write(html) + + def generate_visualizations(self, results): + import matplotlib.pyplot as plt + import seaborn as sns + + # We need dataframes constructed carefully from internal metrics + df = pd.DataFrame(results) + if df.empty: return + + sns.set_theme(style="whitegrid") + base_name = os.path.splitext(self.output_file)[0] if self.output_file else "plots" + + # 1. Bar Chart: WER Before vs After + if HAS_JIWER and 'wer_before' in df.columns: + plt.figure(figsize=(10, 6)) + x = range(len(df)) + width = 0.35 + plt.bar([i - width/2 for i in x], df['wer_before'], width, label='Before Rescoring', color='skyblue') + plt.bar([i + width/2 for i in x], df['wer_after'], width, label='After Rescoring', color='lightgreen') + plt.ylabel('Word Error Rate (WER)') + plt.title('WER Before vs After Shallow Fusion') + plt.xticks(x, df['filename'], rotation=45, ha='right') + plt.legend() + plt.tight_layout() + plt.savefig(os.path.join(self.plots_dir, f"{base_name}_wer_comparison.png"), dpi=300) + plt.close() + + # Compile detailed word-level data for advanced plots + word_data = [] + confusion = {"tp": 0, "fp": 0, "fn": 0, "tn": 0} + + for res in results: + if '_logs' not in res or '_words' not in res or not res.get('_gt'): continue + + logs = res['_logs'] + words = res['_words'] + gt_words = res['_gt'].split() + rescored_map = {log['original'].lower(): log for log in logs} + + for i, w in enumerate(words): + if i >= len(gt_words): break + orig = w['word'].lower() + conf = w['probability'] + gt_w = gt_words[i].lower() + + is_rescored = orig in rescored_map + + if is_rescored: + rep = rescored_map[orig]['replacement'].lower() + if rep == gt_w and orig != gt_w: + confusion['tp'] += 1 # Improved + elif rep != gt_w and orig == gt_w: + confusion['fp'] += 1 # Worsened + elif rep != gt_w and orig != gt_w: + confusion['fn'] += 1 # Stayed bad + else: + if orig == gt_w: + confusion['tn'] += 1 # Stayed good + else: + confusion['fn'] += 1 # Stayed bad + + word_data.append({ + "confidence": conf, + "is_correct": orig == gt_w if not is_rescored else rescored_map[orig]['replacement'].lower() == gt_w, + "rescored": is_rescored, + "position": i / len(words) # normalized position + }) + + if not word_data: return + wdf = pd.DataFrame(word_data) + + # 2. Scatter Plot: Confidence vs Accuracy (Categorical) + plt.figure(figsize=(8, 6)) + sns.stripplot(x="is_correct", y="confidence", data=wdf, hue="is_correct", palette="Set1", jitter=True, alpha=0.5) + plt.title('ASR Confidence vs Final Word Accuracy') + plt.xlabel('Word is Correct (After Fusion)') + plt.ylabel('Whisper Confidence Score') + plt.tight_layout() + plt.savefig(os.path.join(self.plots_dir, f"{base_name}_conf_vs_acc.png"), dpi=300) + plt.close() + + # 3. Histogram: Confidence of Replaced vs Kept + plt.figure(figsize=(8, 6)) + sns.histplot(data=wdf, x="confidence", hue="rescored", multiple="stack", bins=20, palette="viridis") + plt.title('Confidence Distribution: Rescored vs Kept Words') + plt.xlabel('Whisper Confidence Score') + plt.tight_layout() + plt.savefig(os.path.join(self.plots_dir, f"{base_name}_conf_hist.png"), dpi=300) + plt.close() + + # 4. Confusion Matrix Heatmap + plt.figure(figsize=(6, 5)) + cm_matrix = [[confusion['tp'], confusion['fp']], [confusion['fn'], confusion['tn']]] + sns.heatmap(cm_matrix, annot=True, fmt='d', cmap='Blues', + xticklabels=['Correct (Final)', 'Incorrect (Final)'], + yticklabels=['Incorrect (Orig)', 'Correct (Orig)']) + plt.title('Rescoring Confusion Matrix') + plt.ylabel('Original State') + plt.xlabel('Final State') + plt.tight_layout() + plt.savefig(os.path.join(self.plots_dir, f"{base_name}_confusion.png"), dpi=300) + plt.close() + + # 5. Position vs Confidence Heatmap (2D Histogram) + plt.figure(figsize=(8, 6)) + sns.histplot(x=wdf['position'], y=wdf['confidence'], bins=[20, 20], pmax=0.9, cmap="YlGnBu", cbar=True) + plt.title('Word Position vs ASR Confidence') + plt.xlabel('Normalized Position in Audio (0=Start, 1=End)') + plt.ylabel('Confidence Score') + plt.tight_layout() + plt.savefig(os.path.join(self.plots_dir, f"{base_name}_heatmap.png"), dpi=300) + plt.close() + + print(f"Visualizations generated in {self.plots_dir}/") + + def run(self): + self.load_models() + files = [f for f in os.listdir(self.audio_dir) if f.endswith(('.mp3', '.wav', '.m4a'))] + if not files: + print(f"No audio found in {self.audio_dir}") + return + + results = [] + errors = [] + + print(f"Starting evaluation of {len(files)} files...") + for f in tqdm(files, desc="Processing Audio"): + try: + res = self.process_file(f) + results.append(res) + except Exception as e: + print(f"\n[ERROR] Skeping {f}: {e}") + errors.append({"file": f, "error": str(e)}) + + if not results: return + + # Prepare Output + df = pd.DataFrame(results) + + # Identify output paths + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + base_name = os.path.splitext(self.output_file)[0] if self.output_file else f"report_{timestamp}" + + json_path = os.path.join(self.results_dir, f"{base_name}.json") + csv_path = os.path.join(self.results_dir, f"{base_name}.csv") + html_path = os.path.join(self.results_dir, f"{base_name}.html") + + # Export + # Save JSON with internal data (_logs, _words) for later visualization plotting + df.to_json(json_path, orient="records", indent=4) + self.export_csv(df, csv_path) + self.export_html(df, html_path) + self.generate_visualizations(results) + + print("\n=== Evaluation Complete ===") + print(f"Processed: {len(results)} | Errors: {len(errors)}") + if HAS_JIWER and 'wer_before' in df.columns: + wb = df['wer_before'].mean() + wa = df['wer_after'].mean() + print(f"Avg WER Before: {wb:.4f} | Avg WER After: {wa:.4f}") + + print(f"\nReports saved to {self.results_dir}/") + return json_path + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Evaluate Shallow Fusion ASR Pipeline") + parser.add_argument("--config", default="tests/config.yaml", help="Path to config.yaml") + parser.add_argument("--audio_dir", help="Override audio directory") + parser.add_argument("--output", help="Output result basename (e.g. 'run1')") + + args = parser.parse_args() + + runner = EvaluationRunner(config_path=args.config, audio_dir=args.audio_dir, output_file=args.output) + runner.run() diff --git a/tests/ground_truth/sample_lecture.txt b/tests/ground_truth/sample_lecture.txt new file mode 100644 index 0000000..76cdbfb --- /dev/null +++ b/tests/ground_truth/sample_lecture.txt @@ -0,0 +1 @@ +today we are going to talk about linear algebra specifically we will look at how to compute the eigenvalue for a given matrix which is essential for understanding the gaussian distribution in later modules. diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt new file mode 100644 index 0000000..d86a3e8 --- /dev/null +++ b/tests/requirements_test.txt @@ -0,0 +1,5 @@ +pytest +jiwer +psutil +pytest-mock +pytest-cov diff --git a/tests/results/mock_run.csv b/tests/results/mock_run.csv new file mode 100644 index 0000000..4b96a30 --- /dev/null +++ b/tests/results/mock_run.csv @@ -0,0 +1,2 @@ +filename,duration_total_s,latency_rescore_s,latency_per_word_s,throughput_wps,peak_memory_mb,words_total,words_rescored,wer_before,wer_after,wer_improvement,precision,recall,f1,tp,fp,fn +sample_lecture.wav,8.821487426757812e-06,3.0994415283203125e-06,0.01,100,50.0,6,1,0.2,0.0,0.2,1.0,1.0,1.0,1,0,0 diff --git a/tests/results/mock_run.html b/tests/results/mock_run.html new file mode 100644 index 0000000..6a5c481 --- /dev/null +++ b/tests/results/mock_run.html @@ -0,0 +1,44 @@ +Generated: 2026-03-22 02:50:29.217727
| filename | +duration_total_s | +latency_rescore_s | +latency_per_word_s | +throughput_wps | +peak_memory_mb | +words_total | +words_rescored | +wer_before | +wer_after | +wer_improvement | +precision | +recall | +f1 | +tp | +fp | +fn | +
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| sample_lecture.wav | +0.000009 | +0.000003 | +0.01 | +100 | +50.0 | +6 | +1 | +0.2 | +0.0 | +0.2 | +1.0 | +1.0 | +1.0 | +1 | +0 | +0 | +