From 424d38ad1cc3ec5e265debd325a297a6c74fa001 Mon Sep 17 00:00:00 2001 From: Shahin Ramezanzadeh Date: Wed, 18 Feb 2026 17:12:31 +0100 Subject: [PATCH] certainty Delta show --- Data/certainty_show.py | 307 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 306 insertions(+), 1 deletion(-) diff --git a/Data/certainty_show.py b/Data/certainty_show.py index d3d6db0..a35ac75 100644 --- a/Data/certainty_show.py +++ b/Data/certainty_show.py @@ -1214,6 +1214,307 @@ def plot_error_distribution_by_confidence(json_dir_path, ground_truth_path): # plot_error_distribution_by_confidence('jsons_folder/', 'ground_truth.csv') ## + + +# %% Certainty vs Delta +import pandas as pd +import numpy as np +import json +import glob +import os +import matplotlib.pyplot as plt +import seaborn as sns +from matplotlib.patches import Patch +from matplotlib.lines import Line2D + +def plot_confidence_vs_abs_error_refined(json_dir_path, ground_truth_path): + # 1. Load and Merge Data + df_gt = pd.read_csv(ground_truth_path, sep=';') + df_gt['unique_id'] = df_gt['unique_id'].astype(str).str.strip().str.lower() + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str).str.strip().str.lower() + df_gt['EDSS_gt'] = pd.to_numeric(df_gt['EDSS'], errors='coerce') + + all_preds = [] + for file_path in glob.glob(os.path.join(json_dir_path, "*.json")): + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + for entry in data: + if entry.get("success"): + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')).strip().lower(), + 'MedDatum': str(res.get('MedDatum')).strip().lower(), + 'EDSS_pred': pd.to_numeric(res.get('EDSS'), errors='coerce'), + 'confidence': pd.to_numeric(res.get('certainty_percent'), errors='coerce') + }) + + df_merged = pd.merge(pd.DataFrame(all_preds), df_gt[['unique_id', 'MedDatum', 'EDSS_gt']], + on=['unique_id', 'MedDatum'], how='inner').dropna() + + # 2. Calculate Absolute Delta + df_merged['abs_error'] = (df_merged['EDSS_pred'] - df_merged['EDSS_gt']).abs() + + # 3. Binning + bins = [0, 70, 80, 90, 100] + labels = ['Low (<70%)', 'Moderate (70-80%)', 'High (80-90%)', 'Very High (90-100%)'] + df_merged['conf_bin'] = pd.cut(df_merged['confidence'], bins=bins, labels=labels) + + stats = df_merged.groupby('conf_bin', observed=True)['abs_error'].agg(['mean', 'std', 'count']).reset_index() + + # 4. Plotting + plt.figure(figsize=(12, 8)) + # Sequential palette: light to dark + colors = sns.color_palette("Blues", n_colors=len(labels)) + + bars = plt.bar(stats['conf_bin'], stats['mean'], color=colors, edgecolor='black', linewidth=1.2) + + # Standard Error Bars + plt.errorbar(stats['conf_bin'], stats['mean'], + yerr=stats['std']/np.sqrt(stats['count']), + fmt='none', c='black', capsize=6, elinewidth=1.5) + + # Trend Line (Linear Fit) + x_idx = np.arange(len(labels)) + z = np.polyfit(x_idx, stats['mean'], 1) + p = np.poly1d(z) + plt.plot(x_idx, p(x_idx), color="#e74c3c", linestyle="--", linewidth=2.5) + + # 5. THE COMPLETE LEGEND + # We create a specific handle for every single thing on the chart + legend_elements = [ + # Explicit color mapping for bins + Patch(facecolor=colors[0], edgecolor='black', label=f'Bin 1: {labels[0]}'), + Patch(facecolor=colors[1], edgecolor='black', label=f'Bin 2: {labels[1]}'), + Patch(facecolor=colors[2], edgecolor='black', label=f'Bin 3: {labels[2]}'), + Patch(facecolor=colors[3], edgecolor='black', label=f'Bin 4: {labels[3]}'), + # Statistical components + Line2D([0], [0], color='black', marker='_', linestyle='None', markersize=10, label='Standard Error (SEM)'), + Line2D([0], [0], color='#e74c3c', linestyle='--', lw=2.5, label='Correlation Trend (Inverse Rel.)'), + # Metric definition + Patch(color='none', label='Metric: Mean Absolute Error (MAE)') + ] + + plt.legend(handles=legend_elements, loc='upper right', frameon=True, + shadow=True, fontsize=10, title="Legend") + + # Final Labels & Clean-up + plt.title('Validation: Inverse Correlation of Confidence vs. Error Magnitude', fontsize=15, pad=20) + plt.ylabel('Mean Absolute Error (Δ EDSS Points)', fontsize=12) + plt.xlabel('LLM Confidence Bracket', fontsize=12) + + # Text annotations for MAE on bars + for i, bar in enumerate(bars): + plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.05, + f'MAE: {stats.loc[i, "mean"]:.2f}', ha='center', fontweight='bold') + + plt.grid(axis='y', linestyle=':', alpha=0.5) + plt.tight_layout() + plt.show() +## + + + +# %% name +import pandas as pd +import numpy as np +import json +import glob +import os +import matplotlib.pyplot as plt +import seaborn as sns +from matplotlib.patches import Patch +from matplotlib.lines import Line2D + +def plot_final_thesis_error_chart(json_dir_path, ground_truth_path): + # 1. Load Ground Truth & Predictions + df_gt = pd.read_csv(ground_truth_path, sep=';') + df_gt['unique_id'] = df_gt['unique_id'].astype(str).str.strip().str.lower() + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str).str.strip().str.lower() + df_gt['EDSS_gt'] = pd.to_numeric(df_gt['EDSS'], errors='coerce') + + all_preds = [] + for file_path in glob.glob(os.path.join(json_dir_path, "*.json")): + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + for entry in data: + if entry.get("success"): + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')).strip().lower(), + 'MedDatum': str(res.get('MedDatum')).strip().lower(), + 'EDSS_pred': pd.to_numeric(res.get('EDSS'), errors='coerce'), + 'confidence': pd.to_numeric(res.get('certainty_percent'), errors='coerce') + }) + + df_merged = pd.merge(pd.DataFrame(all_preds), df_gt[['unique_id', 'MedDatum', 'EDSS_gt']], + on=['unique_id', 'MedDatum'], how='inner').dropna() + + # 2. Metric Calculation + df_merged['abs_error'] = (df_merged['EDSS_pred'] - df_merged['EDSS_gt']).abs() + + # 3. Binning & Stats + bins = [0, 70, 80, 90, 100] + labels = ['Low (<70%)', 'Moderate (70-80%)', 'High (80-90%)', 'Very High (90-100%)'] + df_merged['conf_bin'] = pd.cut(df_merged['confidence'], bins=bins, labels=labels) + + stats = df_merged.groupby('conf_bin', observed=True)['abs_error'].agg(['mean', 'std', 'count']).reset_index() + + # 4. Plotting + plt.figure(figsize=(13, 8)) + colors = sns.color_palette("Blues", n_colors=len(labels)) + + # BARS (MAE) + bars = plt.bar(stats['conf_bin'], stats['mean'], color=colors, edgecolor='black', alpha=0.85) + + # ERROR BARS (Standard Error of the Mean) + plt.errorbar(stats['conf_bin'], stats['mean'], + yerr=stats['std']/np.sqrt(stats['count']), + fmt='none', c='black', capsize=8, elinewidth=1.5) + + # CORRELATION TREND LINE + x_idx = np.arange(len(labels)) + z = np.polyfit(x_idx, stats['mean'], 1) + p = np.poly1d(z) + plt.plot(x_idx, p(x_idx), color="#e74c3c", linestyle="--", linewidth=3, zorder=5) + + # 5. DATA LABELS (n and MAE) + for i, bar in enumerate(bars): + n_count = int(stats.loc[i, 'count']) + mae_val = stats.loc[i, 'mean'] + plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.04, + f'MAE: {mae_val:.2f}\nn={n_count}', + ha='center', va='bottom', fontweight='bold', fontsize=10) + + # 6. THE COMPLETE LEGEND + legend_elements = [ + Patch(facecolor=colors[0], edgecolor='black', label=f'Bin 1: {labels[0]}'), + Patch(facecolor=colors[1], edgecolor='black', label=f'Bin 2: {labels[1]}'), + Patch(facecolor=colors[2], edgecolor='black', label=f'Bin 3: {labels[2]}'), + Patch(facecolor=colors[3], edgecolor='black', label=f'Bin 4: {labels[3]}'), + Line2D([0], [0], color='#e74c3c', linestyle='--', lw=3, label='Correlation Trend (Inverse Relationship)'), + Line2D([0], [0], color='black', marker='_', linestyle='None', markersize=10, label='Standard Error (SEM)'), + Patch(color='none', label='Metric: Mean Absolute Error (MAE)') + ] + plt.legend(handles=legend_elements, loc='upper right', frameon=True, shadow=True, title="Chart Components") + + # Formatting + plt.title('Clinical Validation: LLM Certainty vs. Prediction Accuracy', fontsize=16, pad=30) + plt.ylabel('Mean Absolute Error (EDSS Points)', fontsize=12) + plt.xlabel('LLM Confidence Bracket', fontsize=12) + plt.grid(axis='y', linestyle=':', alpha=0.5) + plt.ylim(0, stats['mean'].max() + 0.6) # Add room for labels + + plt.tight_layout() + plt.show() + +# plot_final_thesis_error_chart('jsons_folder/', 'gt.csv') +## + + + +# %% 1json +import pandas as pd +import numpy as np +import json +import matplotlib.pyplot as plt +import seaborn as sns +from matplotlib.patches import Patch +from matplotlib.lines import Line2D + +def plot_single_json_error_analysis(json_file_path, ground_truth_path): + # 1. Load Ground Truth + df_gt = pd.read_csv(ground_truth_path, sep=';') + df_gt['unique_id'] = df_gt['unique_id'].astype(str).str.strip().str.lower() + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str).str.strip().str.lower() + df_gt['EDSS_gt'] = pd.to_numeric(df_gt['EDSS'], errors='coerce') + + # 2. Load the Specific JSON + all_preds = [] + with open(json_file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + for entry in data: + if entry.get("success"): + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')).strip().lower(), + 'MedDatum': str(res.get('MedDatum')).strip().lower(), + 'EDSS_pred': pd.to_numeric(res.get('EDSS'), errors='coerce'), + 'confidence': pd.to_numeric(res.get('certainty_percent'), errors='coerce') + }) + + df_pred = pd.DataFrame(all_preds) + + # 3. Merge and Calculate Absolute Error + df_merged = pd.merge(df_pred, df_gt[['unique_id', 'MedDatum', 'EDSS_gt']], + on=['unique_id', 'MedDatum'], how='inner').dropna() + + df_merged['abs_error'] = (df_merged['EDSS_pred'] - df_merged['EDSS_gt']).abs() + + # 4. Binning and Statistics + bins = [0, 70, 80, 90, 100] + labels = ['Low (<70%)', 'Moderate (70-80%)', 'High (80-90%)', 'Very High (90-100%)'] + df_merged['conf_bin'] = pd.cut(df_merged['confidence'], bins=bins, labels=labels) + + stats = df_merged.groupby('conf_bin', observed=True)['abs_error'].agg(['mean', 'std', 'count']).reset_index() + + # 5. Plotting + plt.figure(figsize=(13, 8)) + colors = sns.color_palette("Blues", n_colors=len(labels)) + + # BARS (MAE) + bars = plt.bar(stats['conf_bin'], stats['mean'], color=colors, edgecolor='black', alpha=0.85) + + # ERROR BARS (SEM) + plt.errorbar(stats['conf_bin'], stats['mean'], + yerr=stats['std']/np.sqrt(stats['count']), + fmt='none', c='black', capsize=8, elinewidth=1.5) + + # CORRELATION TREND LINE + x_idx = np.arange(len(labels)) + z = np.polyfit(x_idx, stats['mean'], 1) + p = np.poly1d(z) + plt.plot(x_idx, p(x_idx), color="#e74c3c", linestyle="--", linewidth=3, zorder=5) + + # 6. DATA LABELS (n and MAE) + for i, bar in enumerate(bars): + n_count = int(stats.loc[i, 'count']) + mae_val = stats.loc[i, 'mean'] + plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.04, + f'MAE: {mae_val:.2f}\nn={n_count}', + ha='center', va='bottom', fontweight='bold', fontsize=10) + + # 7. COMPREHENSIVE LEGEND + legend_elements = [ + Patch(facecolor=colors[0], edgecolor='black', label=f'Bin 1: {labels[0]}'), + Patch(facecolor=colors[1], edgecolor='black', label=f'Bin 2: {labels[1]}'), + Patch(facecolor=colors[2], edgecolor='black', label=f'Bin 3: {labels[2]}'), + Patch(facecolor=colors[3], edgecolor='black', label=f'Bin 4: {labels[3]}'), + Line2D([0], [0], color='#e74c3c', linestyle='--', lw=3, label='Inverse Trend Line'), + Line2D([0], [0], color='black', marker='_', linestyle='None', markersize=10, label='Std Error (SEM)'), + Patch(color='none', label='Metric: Mean Absolute Error (MAE)') + ] + plt.legend(handles=legend_elements, loc='upper right', frameon=True, shadow=True, title="Legend") + + # Final Styling + plt.title('Validation: Confidence vs. Error Magnitude (Iteration 1 Only)', fontsize=15, pad=30) + plt.ylabel('Mean Absolute Error (EDSS Points)', fontsize=12) + plt.xlabel('LLM Confidence Bracket', fontsize=12) + plt.grid(axis='y', linestyle=':', alpha=0.5) + plt.ylim(0, stats['mean'].max() + 0.6) + + plt.tight_layout() + plt.show() + +# --- RUN THE PLOT --- +json_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json" +gt_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv" + +plot_single_json_error_analysis(json_path, gt_path) +## + + + + # %% Usage # --- Usage --- @@ -1231,5 +1532,9 @@ def plot_error_distribution_by_confidence(json_dir_path, ground_truth_path): #plot_edss_vs_confidence_boxplot('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration') #plot_gt_vs_llm_boxplot('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') #plot_categorical_vs_categorical('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') -plot_error_distribution_by_confidence('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') +#plot_error_distribution_by_confidence('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') +#plot_confidence_vs_abs_error_refined('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') +#plot_confidence_vs_abs_error_with_counts('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') +plot_final_thesis_error_chart('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') + ##