2372 lines
86 KiB
Python
2372 lines
86 KiB
Python
# %% Confirm EDSS missing
|
||
import pandas as pd
|
||
import numpy as np
|
||
|
||
def clean_series(s):
|
||
return s.astype(str).str.strip().str.lower()
|
||
|
||
def gt_edss_audit(ground_truth_path, edss_col="EDSS"):
|
||
df_gt = pd.read_csv(ground_truth_path, sep=';')
|
||
|
||
# normalize keys
|
||
df_gt['unique_id'] = clean_series(df_gt['unique_id'])
|
||
df_gt['MedDatum'] = clean_series(df_gt['MedDatum'])
|
||
df_gt['key'] = df_gt['unique_id'] + "_" + df_gt['MedDatum']
|
||
|
||
print("GT rows:", len(df_gt))
|
||
print("GT unique keys:", df_gt['key'].nunique())
|
||
|
||
# IMPORTANT: parse EDSS robustly (German decimal commas etc.)
|
||
if edss_col in df_gt.columns:
|
||
edss_raw = df_gt[edss_col]
|
||
edss_num = pd.to_numeric(
|
||
edss_raw.astype(str).str.replace(",", ".", regex=False).str.strip(),
|
||
errors="coerce"
|
||
)
|
||
df_gt["_edss_num"] = edss_num
|
||
|
||
print(f"GT missing EDSS look (numeric-coerce): {df_gt['_edss_num'].isna().sum()}")
|
||
print(f"GT missing EDSS unique keys: {df_gt.loc[df_gt['_edss_num'].isna(), 'key'].nunique()}")
|
||
|
||
# duplicates on key
|
||
dup = df_gt['key'].duplicated(keep=False)
|
||
print("GT duplicate-key rows:", dup.sum())
|
||
if dup.any():
|
||
# how many duplicate keys exist?
|
||
print("GT duplicate keys:", df_gt.loc[dup, 'key'].nunique())
|
||
# of duplicate-key rows, how many have missing EDSS?
|
||
print("Duplicate-key rows with missing EDSS:", df_gt.loc[dup, "_edss_num"].isna().sum())
|
||
|
||
# show the worst offenders
|
||
print("\nTop duplicate keys (by count):")
|
||
print(df_gt.loc[dup, 'key'].value_counts().head(10))
|
||
else:
|
||
print(f"EDSS column '{edss_col}' not found in GT columns:", df_gt.columns.tolist())
|
||
|
||
return df_gt
|
||
|
||
df_gt = gt_edss_audit("/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv", edss_col="EDSS")
|
||
|
||
##
|
||
|
||
|
||
|
||
|
||
# %% trace missing ones
|
||
|
||
import json, glob, os
|
||
import pandas as pd
|
||
|
||
def load_preds(json_dir_path):
|
||
all_preds = []
|
||
for file_path in glob.glob(os.path.join(json_dir_path, "*.json")):
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
file_name = os.path.basename(file_path)
|
||
for entry in data:
|
||
if entry.get("success"):
|
||
res = entry["result"]
|
||
all_preds.append({
|
||
"unique_id": str(res.get("unique_id")).strip().lower(),
|
||
"MedDatum": str(res.get("MedDatum")).strip().lower(),
|
||
"file": file_name
|
||
})
|
||
df_pred = pd.DataFrame(all_preds)
|
||
df_pred["key"] = df_pred["unique_id"] + "_" + df_pred["MedDatum"]
|
||
return df_pred
|
||
|
||
df_pred = load_preds("/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration")
|
||
print("Pred rows:", len(df_pred))
|
||
print("Pred unique keys:", df_pred["key"].nunique())
|
||
|
||
# Suppose df_gt was returned from step 1 and has _edss_num + key
|
||
missing_gt_keys = set(df_gt.loc[df_gt["_edss_num"].isna(), "key"])
|
||
|
||
df_pred["gt_key_missing_edss"] = df_pred["key"].isin(missing_gt_keys)
|
||
|
||
print("Pred rows whose GT key has missing EDSS:", df_pred["gt_key_missing_edss"].sum())
|
||
print("Unique keys (among preds) whose GT EDSS missing:", df_pred.loc[df_pred["gt_key_missing_edss"], "key"].nunique())
|
||
|
||
print("\nTop files contributing to missing-GT-EDSS rows:")
|
||
print(df_pred.loc[df_pred["gt_key_missing_edss"], "file"].value_counts().head(20))
|
||
|
||
print("\nTop keys replicated in predictions (why count inflates):")
|
||
print(df_pred.loc[df_pred["gt_key_missing_edss"], "key"].value_counts().head(20))
|
||
|
||
|
||
##
|
||
|
||
|
||
# %% verify
|
||
|
||
merged = df_pred.merge(
|
||
df_gt[["key", "_edss_num"]], # use the numeric-coerced GT EDSS
|
||
on="key",
|
||
how="left",
|
||
validate="many_to_one" # will ERROR if GT has duplicate keys (GOOD!)
|
||
)
|
||
|
||
print("Merged rows:", len(merged))
|
||
print("Merged missing GT EDSS:", merged["_edss_num"].isna().sum())
|
||
|
||
|
||
##
|
||
|
||
|
||
# %% 1json (rewritten with robust parsing + detailed data log)
|
||
import pandas as pd
|
||
import numpy as np
|
||
import json
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
from matplotlib.patches import Patch
|
||
from matplotlib.lines import Line2D
|
||
|
||
def plot_single_json_error_analysis_with_log(
|
||
json_file_path,
|
||
ground_truth_path,
|
||
edss_gt_col="EDSS",
|
||
min_bin_count=5,
|
||
):
|
||
def norm_str(x):
|
||
# normalize identifiers and dates consistently
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
# robust numeric parse: handles "3,5" as 3.5, blanks, "nan", etc.
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
print("\n" + "="*80)
|
||
print("SINGLE-JSON ERROR ANALYSIS (WITH LOG)")
|
||
print("="*80)
|
||
print(f"JSON: {json_file_path}")
|
||
print(f"GT: {ground_truth_path}")
|
||
|
||
# ------------------------------------------------------------------
|
||
# 1) Load Ground Truth
|
||
# ------------------------------------------------------------------
|
||
df_gt = pd.read_csv(ground_truth_path, sep=";")
|
||
|
||
required_gt_cols = {"unique_id", "MedDatum", edss_gt_col}
|
||
missing_cols = required_gt_cols - set(df_gt.columns)
|
||
if missing_cols:
|
||
raise ValueError(f"GT is missing required columns: {missing_cols}. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
|
||
# Robust EDSS parsing (important!)
|
||
df_gt["EDSS_gt"] = df_gt[edss_gt_col].map(parse_edss)
|
||
|
||
# GT logs
|
||
print("\n--- GT LOG ---")
|
||
print(f"GT rows: {len(df_gt)}")
|
||
print(f"GT unique keys: {df_gt['key'].nunique()}")
|
||
gt_dup = df_gt["key"].duplicated(keep=False).sum()
|
||
print(f"GT duplicate-key rows: {gt_dup}")
|
||
print(f"GT missing EDSS (numeric): {df_gt['EDSS_gt'].isna().sum()}")
|
||
print(f"GT missing EDSS unique keys: {df_gt.loc[df_gt['EDSS_gt'].isna(), 'key'].nunique()}")
|
||
|
||
if gt_dup > 0:
|
||
print("\n[WARNING] GT has duplicate keys. Merge can duplicate rows. Example duplicate keys:")
|
||
print(df_gt.loc[df_gt["key"].duplicated(keep=False), "key"].value_counts().head(10))
|
||
|
||
# ------------------------------------------------------------------
|
||
# 2) Load Predictions from the specific JSON
|
||
# ------------------------------------------------------------------
|
||
with open(json_file_path, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
|
||
total_entries = len(data)
|
||
success_entries = sum(1 for e in data if e.get("success"))
|
||
|
||
all_preds = []
|
||
skipped = {
|
||
"not_success": 0,
|
||
"missing_uid_or_date": 0,
|
||
"missing_edss": 0,
|
||
"missing_conf": 0,
|
||
}
|
||
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
skipped["not_success"] += 1
|
||
continue
|
||
|
||
res = entry.get("result", {})
|
||
uid = res.get("unique_id")
|
||
md = res.get("MedDatum")
|
||
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
skipped["missing_uid_or_date"] += 1
|
||
continue
|
||
|
||
edss_pred = parse_edss(res.get("EDSS"))
|
||
conf = pd.to_numeric(res.get("certainty_percent"), errors="coerce")
|
||
|
||
if pd.isna(edss_pred):
|
||
skipped["missing_edss"] += 1
|
||
if pd.isna(conf):
|
||
skipped["missing_conf"] += 1
|
||
|
||
all_preds.append({
|
||
"unique_id": norm_str(uid),
|
||
"MedDatum": norm_str(md),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": edss_pred,
|
||
"confidence": conf,
|
||
})
|
||
|
||
df_pred = pd.DataFrame(all_preds)
|
||
|
||
# Pred logs
|
||
print("\n--- PRED LOG ---")
|
||
print(f"JSON total entries: {total_entries}")
|
||
print(f"JSON success entries: {success_entries}")
|
||
print(f"Pred rows loaded (success + has keys): {len(df_pred)}")
|
||
if len(df_pred) == 0:
|
||
print("[ERROR] No usable prediction rows found. Nothing to plot.")
|
||
return
|
||
|
||
print(f"Pred unique keys: {df_pred['key'].nunique()}")
|
||
print(f"Pred missing EDSS (numeric): {df_pred['EDSS_pred'].isna().sum()}")
|
||
print(f"Pred missing confidence: {df_pred['confidence'].isna().sum()}")
|
||
print("Skipped counts:", skipped)
|
||
|
||
# Are keys duplicated within this JSON? (often yes if multiple notes map to same key)
|
||
key_counts = df_pred["key"].value_counts()
|
||
dup_pred_rows = (key_counts > 1).sum()
|
||
max_rep = int(key_counts.max())
|
||
print(f"Keys with >1 prediction in this JSON: {dup_pred_rows}")
|
||
print(f"Max repetitions of a single key in this JSON: {max_rep}")
|
||
if max_rep > 1:
|
||
print("Top repeated keys in this JSON:")
|
||
print(key_counts.head(10))
|
||
|
||
# ------------------------------------------------------------------
|
||
# 3) Merge (and diagnose why rows drop)
|
||
# ------------------------------------------------------------------
|
||
# Diagnose how many pred keys exist in GT
|
||
gt_key_set = set(df_gt["key"])
|
||
df_pred["key_in_gt"] = df_pred["key"].isin(gt_key_set)
|
||
not_in_gt = df_pred.loc[~df_pred["key_in_gt"]]
|
||
|
||
print("\n--- KEY MATCH LOG ---")
|
||
print(f"Pred rows with key found in GT: {df_pred['key_in_gt'].sum()} / {len(df_pred)}")
|
||
print(f"Pred rows with key NOT found in GT: {len(not_in_gt)}")
|
||
if len(not_in_gt) > 0:
|
||
print("[WARNING] Some prediction keys are not present in GT. First 10:")
|
||
print(not_in_gt[["unique_id", "MedDatum", "key"]].head(10))
|
||
|
||
# Now merge; we expect GT is one-to-many with pred (many_to_one)
|
||
# If GT had duplicates, validate would raise.
|
||
df_merged = df_pred.merge(
|
||
df_gt[["key", "EDSS_gt"]],
|
||
on="key",
|
||
how="inner",
|
||
validate="many_to_one"
|
||
)
|
||
|
||
print("\n--- MERGE LOG ---")
|
||
print(f"Merged rows (inner join): {len(df_merged)}")
|
||
print(f"Merged unique keys: {df_merged['key'].nunique()}")
|
||
print(f"Merged missing GT EDSS: {df_merged['EDSS_gt'].isna().sum()}")
|
||
print(f"Merged missing pred EDSS: {df_merged['EDSS_pred'].isna().sum()}")
|
||
print(f"Merged missing confidence:{df_merged['confidence'].isna().sum()}")
|
||
|
||
# How many rows will be removed by dropna() in your old code?
|
||
# Old code did .dropna() on ALL columns, which can remove rows for missing confidence too.
|
||
rows_complete = df_merged.dropna(subset=["EDSS_gt", "EDSS_pred", "confidence"])
|
||
print("\n--- FILTER LOG (what will be used for stats/plot) ---")
|
||
print(f"Rows with all required fields (EDSS_gt, EDSS_pred, confidence): {len(rows_complete)}")
|
||
if len(rows_complete) == 0:
|
||
print("[ERROR] No complete rows after filtering. Nothing to plot.")
|
||
return
|
||
|
||
# Compute abs error
|
||
rows_complete = rows_complete.copy()
|
||
rows_complete["abs_error"] = (rows_complete["EDSS_pred"] - rows_complete["EDSS_gt"]).abs()
|
||
|
||
# ------------------------------------------------------------------
|
||
# 4) Binning + stats (with guardrails)
|
||
# ------------------------------------------------------------------
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
|
||
# Confidence outside bins becomes NaN; log it
|
||
rows_complete["conf_bin"] = pd.cut(rows_complete["confidence"], bins=bins, labels=labels, include_lowest=True)
|
||
conf_outside = rows_complete["conf_bin"].isna().sum()
|
||
print(f"Rows with confidence outside [0,100] or outside bin edges: {conf_outside}")
|
||
if conf_outside > 0:
|
||
print("Example confidences outside bins:")
|
||
print(rows_complete.loc[rows_complete["conf_bin"].isna(), "confidence"].head(20).to_list())
|
||
|
||
df_plot = rows_complete.dropna(subset=["conf_bin"])
|
||
stats = (
|
||
df_plot.groupby("conf_bin", observed=True)["abs_error"]
|
||
.agg(mean="mean", std="std", count="count")
|
||
.reindex(labels)
|
||
.reset_index()
|
||
)
|
||
|
||
print("\n--- BIN STATS ---")
|
||
print(stats)
|
||
|
||
# Warn about low counts
|
||
low_bins = stats.loc[stats["count"].fillna(0) < min_bin_count, ["conf_bin", "count"]]
|
||
if not low_bins.empty:
|
||
print(f"\n[WARNING] Some bins have < {min_bin_count} rows; error bars/trend may be unstable:")
|
||
print(low_bins)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 5) Plot
|
||
# ------------------------------------------------------------------
|
||
plt.figure(figsize=(13, 8))
|
||
colors = sns.color_palette("Blues", n_colors=len(labels))
|
||
|
||
# Replace NaNs in mean for plotting bars (empty bins)
|
||
means = stats["mean"].to_numpy()
|
||
counts = stats["count"].fillna(0).astype(int).to_numpy()
|
||
stds = stats["std"].to_numpy()
|
||
|
||
# For bins with no data, bar height 0 (and no errorbar)
|
||
means_plot = np.nan_to_num(means, nan=0.0)
|
||
|
||
bars = plt.bar(labels, means_plot, color=colors, edgecolor="black", alpha=0.85)
|
||
|
||
# Error bars only where count>1 and std is not NaN
|
||
sem = np.where((counts > 1) & (~np.isnan(stds)), stds / np.sqrt(counts), np.nan)
|
||
plt.errorbar(labels, means_plot, yerr=sem, fmt="none", c="black", capsize=8, elinewidth=1.5)
|
||
|
||
# Trend line only if at least 2 non-empty bins
|
||
valid_idx = np.where(~np.isnan(means))[0]
|
||
if len(valid_idx) >= 2:
|
||
x_idx = np.arange(len(labels))
|
||
z = np.polyfit(valid_idx, means[valid_idx], 1)
|
||
p = np.poly1d(z)
|
||
plt.plot(x_idx, p(x_idx), color="#e74c3c", linestyle="--", linewidth=3, zorder=5)
|
||
trend_label = "Trend Line"
|
||
else:
|
||
trend_label = "Trend Line (insufficient bins)"
|
||
print("\n[INFO] Not enough non-empty bins to fit a trend line.")
|
||
|
||
# Data labels
|
||
for i, bar in enumerate(bars):
|
||
n_count = int(counts[i])
|
||
mae_val = means[i]
|
||
if np.isnan(mae_val) or n_count == 0:
|
||
txt = "empty"
|
||
y = 0.02
|
||
else:
|
||
txt = f"MAE: {mae_val:.2f}\nn={n_count}"
|
||
y = bar.get_height() + 0.04
|
||
plt.text(
|
||
bar.get_x() + bar.get_width()/2,
|
||
y,
|
||
txt,
|
||
ha="center",
|
||
va="bottom",
|
||
fontweight="bold",
|
||
fontsize=10
|
||
)
|
||
|
||
# Legend
|
||
legend_elements = [
|
||
Patch(facecolor=colors[0], edgecolor="black", label=f"Bin 1: {labels[0]}"),
|
||
Patch(facecolor=colors[1], edgecolor="black", label=f"Bin 2: {labels[1]}"),
|
||
Patch(facecolor=colors[2], edgecolor="black", label=f"Bin 3: {labels[2]}"),
|
||
Patch(facecolor=colors[3], edgecolor="black", label=f"Bin 4: {labels[3]}"),
|
||
Line2D([0], [0], color="#e74c3c", linestyle="--", lw=3, label=trend_label),
|
||
Line2D([0], [0], color="black", marker="_", linestyle="None", markersize=10, label="Std Error (SEM)"),
|
||
Patch(color="none", label="Metric: Mean Absolute Error (MAE)")
|
||
]
|
||
plt.legend(handles=legend_elements, loc="upper right", frameon=True, shadow=True, title="Legend")
|
||
|
||
plt.title("Validation: Confidence vs. Error Magnitude (Single JSON)", fontsize=15, pad=30)
|
||
plt.ylabel("Mean Absolute Error (EDSS Points)", fontsize=12)
|
||
plt.xlabel("LLM Confidence Bracket", fontsize=12)
|
||
plt.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
ymax = np.nanmax(means) if np.any(~np.isnan(means)) else 0.0
|
||
plt.ylim(0, max(0.5, float(ymax) + 0.6))
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
print("\n" + "="*80)
|
||
print("DONE")
|
||
print("="*80)
|
||
|
||
|
||
# --- RUN ---
|
||
json_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json"
|
||
gt_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv"
|
||
|
||
plot_single_json_error_analysis_with_log(json_path, gt_path)
|
||
|
||
|
||
|
||
##
|
||
|
||
|
||
# %% Certainty vs Delta (rewritten with robust parsing + detailed data loss logs)
|
||
import pandas as pd
|
||
import numpy as np
|
||
import json
|
||
import glob
|
||
import os
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
from matplotlib.patches import Patch
|
||
from matplotlib.lines import Line2D
|
||
|
||
def plot_confidence_vs_abs_error_with_log(
|
||
json_dir_path,
|
||
ground_truth_path,
|
||
edss_gt_col="EDSS",
|
||
min_bin_count=5,
|
||
include_lowest=True,
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
# robust numeric parse: handles comma decimals and empty tokens
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
print("\n" + "="*90)
|
||
print("CERTAINTY vs ABS ERROR (ALL JSONs) — WITH DATA LOSS LOG")
|
||
print("="*90)
|
||
print(f"JSON DIR: {json_dir_path}")
|
||
print(f"GT FILE: {ground_truth_path}")
|
||
|
||
# ------------------------------------------------------------------
|
||
# 1) Load GT
|
||
# ------------------------------------------------------------------
|
||
df_gt = pd.read_csv(ground_truth_path, sep=";")
|
||
required_gt_cols = {"unique_id", "MedDatum", edss_gt_col}
|
||
missing_cols = required_gt_cols - set(df_gt.columns)
|
||
if missing_cols:
|
||
raise ValueError(f"GT missing columns: {missing_cols}. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[edss_gt_col].map(parse_edss)
|
||
|
||
# GT logs
|
||
print("\n--- GT LOG ---")
|
||
print(f"GT rows: {len(df_gt)}")
|
||
print(f"GT unique keys: {df_gt['key'].nunique()}")
|
||
gt_dup_rows = df_gt["key"].duplicated(keep=False).sum()
|
||
print(f"GT duplicate-key rows: {gt_dup_rows}")
|
||
print(f"GT missing EDSS (numeric): {df_gt['EDSS_gt'].isna().sum()}")
|
||
print(f"GT missing EDSS unique keys: {df_gt.loc[df_gt['EDSS_gt'].isna(), 'key'].nunique()}")
|
||
if gt_dup_rows > 0:
|
||
print("\n[WARNING] GT has duplicate keys; merge can explode rows. Top duplicate keys:")
|
||
print(df_gt.loc[df_gt["key"].duplicated(keep=False), "key"].value_counts().head(10))
|
||
|
||
gt_key_set = set(df_gt["key"])
|
||
|
||
# ------------------------------------------------------------------
|
||
# 2) Load predictions from all JSON files (with per-file logs)
|
||
# ------------------------------------------------------------------
|
||
json_files = sorted(glob.glob(os.path.join(json_dir_path, "*.json")))
|
||
if not json_files:
|
||
raise FileNotFoundError(f"No JSON files found in: {json_dir_path}")
|
||
|
||
all_preds = []
|
||
per_file_summary = []
|
||
|
||
total_entries_all = 0
|
||
total_success_all = 0
|
||
skipped_all = {"not_success": 0, "missing_uid_or_date": 0}
|
||
|
||
for file_path in json_files:
|
||
with open(file_path, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
|
||
total_entries = len(data)
|
||
success_entries = sum(1 for e in data if e.get("success"))
|
||
|
||
total_entries_all += total_entries
|
||
total_success_all += success_entries
|
||
|
||
skipped = {"not_success": 0, "missing_uid_or_date": 0}
|
||
loaded_rows = 0
|
||
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
skipped["not_success"] += 1
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid = res.get("unique_id")
|
||
md = res.get("MedDatum")
|
||
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
skipped["missing_uid_or_date"] += 1
|
||
continue
|
||
|
||
all_preds.append({
|
||
"file": os.path.basename(file_path),
|
||
"unique_id": norm_str(uid),
|
||
"MedDatum": norm_str(md),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
"confidence": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
loaded_rows += 1
|
||
|
||
skipped_all["not_success"] += skipped["not_success"]
|
||
skipped_all["missing_uid_or_date"] += skipped["missing_uid_or_date"]
|
||
|
||
per_file_summary.append({
|
||
"file": os.path.basename(file_path),
|
||
"entries_total": total_entries,
|
||
"entries_success": success_entries,
|
||
"pred_rows_loaded": loaded_rows,
|
||
"skipped_not_success": skipped["not_success"],
|
||
"skipped_missing_uid_or_date": skipped["missing_uid_or_date"],
|
||
})
|
||
|
||
df_pred = pd.DataFrame(all_preds)
|
||
df_file = pd.DataFrame(per_file_summary)
|
||
|
||
# PRED logs
|
||
print("\n--- PRED LOG (ALL FILES) ---")
|
||
print(f"JSON files found: {len(json_files)}")
|
||
print(f"Total JSON entries: {total_entries_all}")
|
||
print(f"Total success entries:{total_success_all}")
|
||
print(f"Pred rows loaded (success + has keys): {len(df_pred)}")
|
||
if len(df_pred) == 0:
|
||
print("[ERROR] No usable prediction rows found. Nothing to plot.")
|
||
return
|
||
|
||
print(f"Pred unique keys (across all files): {df_pred['key'].nunique()}")
|
||
print(f"Pred missing EDSS (numeric): {df_pred['EDSS_pred'].isna().sum()}")
|
||
print(f"Pred missing confidence: {df_pred['confidence'].isna().sum()}")
|
||
print("Skipped totals:", skipped_all)
|
||
|
||
# show per-file quick check (useful when one iteration is broken)
|
||
print("\nPer-file loaded rows (head):")
|
||
print(df_file.sort_values("file").head(10))
|
||
|
||
# ------------------------------------------------------------------
|
||
# 3) Key match log (pred -> GT)
|
||
# ------------------------------------------------------------------
|
||
df_pred["key_in_gt"] = df_pred["key"].isin(gt_key_set)
|
||
not_in_gt = df_pred.loc[~df_pred["key_in_gt"]]
|
||
|
||
print("\n--- KEY MATCH LOG ---")
|
||
print(f"Pred rows with key found in GT: {df_pred['key_in_gt'].sum()} / {len(df_pred)}")
|
||
print(f"Pred rows with key NOT in GT: {len(not_in_gt)}")
|
||
if len(not_in_gt) > 0:
|
||
print("[WARNING] Example keys not found in GT (first 10):")
|
||
print(not_in_gt[["file", "unique_id", "MedDatum", "key"]].head(10))
|
||
print("\n[WARNING] Files contributing most to key-mismatch:")
|
||
print(not_in_gt["file"].value_counts().head(10))
|
||
|
||
# ------------------------------------------------------------------
|
||
# 4) Merge (no dropna yet) + detailed data loss accounting
|
||
# ------------------------------------------------------------------
|
||
df_merged = df_pred.merge(
|
||
df_gt[["key", "EDSS_gt"]],
|
||
on="key",
|
||
how="inner",
|
||
validate="many_to_one" # catches GT duplicates
|
||
)
|
||
|
||
print("\n--- MERGE LOG ---")
|
||
print(f"Merged rows (inner join): {len(df_merged)}")
|
||
print(f"Merged unique keys: {df_merged['key'].nunique()}")
|
||
|
||
# Now quantify what you lose at each filter stage
|
||
n0 = len(df_merged)
|
||
|
||
miss_gt = df_merged["EDSS_gt"].isna()
|
||
miss_pred = df_merged["EDSS_pred"].isna()
|
||
miss_conf = df_merged["confidence"].isna()
|
||
|
||
print("\n--- MISSINGNESS IN MERGED ---")
|
||
print(f"Missing GT EDSS: {miss_gt.sum()}")
|
||
print(f"Missing Pred EDSS: {miss_pred.sum()}")
|
||
print(f"Missing Confidence: {miss_conf.sum()}")
|
||
|
||
# IMPORTANT: your old code used .dropna() with no subset => drops if ANY column is NaN.
|
||
# We'll replicate the intended logic explicitly and log counts.
|
||
df_complete = df_merged.dropna(subset=["EDSS_gt", "EDSS_pred", "confidence"])
|
||
n1 = len(df_complete)
|
||
print("\n--- FILTER LOG ---")
|
||
print(f"Rows before filtering: {n0}")
|
||
print(f"Rows after requiring EDSS_gt, EDSS_pred, confidence: {n1}")
|
||
print(f"Rows lost due to missing required fields: {n0 - n1}")
|
||
|
||
# Break down why rows were lost (overlap-aware)
|
||
lost_mask = df_merged[["EDSS_gt", "EDSS_pred", "confidence"]].isna().any(axis=1)
|
||
lost = df_merged.loc[lost_mask].copy()
|
||
if len(lost) > 0:
|
||
lost_reason = (
|
||
(lost["EDSS_gt"].isna()).astype(int).map({1:"GT",0:""}) +
|
||
(lost["EDSS_pred"].isna()).astype(int).map({1:"+PRED",0:""}) +
|
||
(lost["confidence"].isna()).astype(int).map({1:"+CONF",0:""})
|
||
)
|
||
lost["loss_reason"] = lost_reason.str.replace(r"^\+", "", regex=True).replace("", "UNKNOWN")
|
||
print("\nTop loss reasons (overlap-aware):")
|
||
print(lost["loss_reason"].value_counts().head(10))
|
||
|
||
print("\nFiles contributing most to lost rows:")
|
||
print(lost["file"].value_counts().head(10))
|
||
|
||
if len(df_complete) == 0:
|
||
print("[ERROR] No complete rows left after filtering. Nothing to plot.")
|
||
return
|
||
|
||
# ------------------------------------------------------------------
|
||
# 5) Abs error + binning
|
||
# ------------------------------------------------------------------
|
||
df_complete = df_complete.copy()
|
||
df_complete["abs_error"] = (df_complete["EDSS_pred"] - df_complete["EDSS_gt"]).abs()
|
||
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
|
||
df_complete["conf_bin"] = pd.cut(
|
||
df_complete["confidence"],
|
||
bins=bins,
|
||
labels=labels,
|
||
include_lowest=include_lowest
|
||
)
|
||
|
||
conf_outside = df_complete["conf_bin"].isna().sum()
|
||
print("\n--- BINNING LOG ---")
|
||
print(f"Rows with confidence outside bin edges / invalid: {conf_outside}")
|
||
if conf_outside > 0:
|
||
print("Example out-of-bin confidences:")
|
||
print(df_complete.loc[df_complete["conf_bin"].isna(), "confidence"].head(20).to_list())
|
||
|
||
df_plot = df_complete.dropna(subset=["conf_bin"])
|
||
print(f"Rows kept for bin stats/plot (after dropping out-of-bin): {len(df_plot)}")
|
||
print(f"Rows lost due to out-of-bin confidence: {len(df_complete) - len(df_plot)}")
|
||
|
||
stats = (
|
||
df_plot.groupby("conf_bin", observed=True)["abs_error"]
|
||
.agg(mean="mean", std="std", count="count")
|
||
.reindex(labels)
|
||
.reset_index()
|
||
)
|
||
|
||
print("\n--- BIN STATS ---")
|
||
print(stats)
|
||
|
||
low_bins = stats.loc[stats["count"].fillna(0) < min_bin_count, ["conf_bin", "count"]]
|
||
if not low_bins.empty:
|
||
print(f"\n[WARNING] Some bins have < {min_bin_count} rows (unstable SEM/trend):")
|
||
print(low_bins)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 6) Plot
|
||
# ------------------------------------------------------------------
|
||
plt.figure(figsize=(12, 8))
|
||
colors = sns.color_palette("Blues", n_colors=len(labels))
|
||
|
||
means = stats["mean"].to_numpy()
|
||
counts = stats["count"].fillna(0).astype(int).to_numpy()
|
||
stds = stats["std"].to_numpy()
|
||
|
||
means_plot = np.nan_to_num(means, nan=0.0)
|
||
bars = plt.bar(labels, means_plot, color=colors, edgecolor="black", linewidth=1.2)
|
||
|
||
sem = np.where((counts > 1) & (~np.isnan(stds)), stds / np.sqrt(counts), np.nan)
|
||
plt.errorbar(labels, means_plot, yerr=sem, fmt="none", c="black", capsize=6, elinewidth=1.5)
|
||
|
||
# Trend line only if >=2 non-empty bins
|
||
valid_idx = np.where(~np.isnan(means))[0]
|
||
if len(valid_idx) >= 2:
|
||
x_idx = np.arange(len(labels))
|
||
z = np.polyfit(valid_idx, means[valid_idx], 1)
|
||
p = np.poly1d(z)
|
||
plt.plot(x_idx, p(x_idx), color="#e74c3c", linestyle="--", linewidth=2.5)
|
||
trend_label = "Correlation Trend"
|
||
else:
|
||
trend_label = "Correlation Trend (insufficient bins)"
|
||
print("\n[INFO] Not enough non-empty bins to fit a trend line.")
|
||
|
||
# Bar annotations (MAE + n)
|
||
for i, bar in enumerate(bars):
|
||
n = int(counts[i])
|
||
m = means[i]
|
||
if n == 0 or np.isnan(m):
|
||
txt = "empty"
|
||
y = 0.02
|
||
else:
|
||
txt = f"MAE: {m:.2f}\nn={n}"
|
||
y = bar.get_height() + 0.05
|
||
plt.text(bar.get_x() + bar.get_width()/2, y, txt, ha="center", fontweight="bold")
|
||
|
||
legend_elements = [
|
||
Patch(facecolor=colors[0], edgecolor="black", label=f"Bin 1: {labels[0]}"),
|
||
Patch(facecolor=colors[1], edgecolor="black", label=f"Bin 2: {labels[1]}"),
|
||
Patch(facecolor=colors[2], edgecolor="black", label=f"Bin 3: {labels[2]}"),
|
||
Patch(facecolor=colors[3], edgecolor="black", label=f"Bin 4: {labels[3]}"),
|
||
Line2D([0], [0], color="black", marker="_", linestyle="None", markersize=10, label="Standard Error (SEM)"),
|
||
Line2D([0], [0], color="#e74c3c", linestyle="--", lw=2.5, label=trend_label),
|
||
Patch(color="none", label="Metric: Mean Absolute Error (MAE)")
|
||
]
|
||
plt.legend(handles=legend_elements, loc="upper right", frameon=True, shadow=True, fontsize=10, title="Legend")
|
||
|
||
plt.title("Validation: Inverse Correlation of Confidence vs. Error Magnitude", fontsize=15, pad=20)
|
||
plt.ylabel("Mean Absolute Error (Δ EDSS Points)", fontsize=12)
|
||
plt.xlabel("LLM Confidence Bracket", fontsize=12)
|
||
plt.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
ymax = np.nanmax(means) if np.any(~np.isnan(means)) else 0.0
|
||
plt.ylim(0, max(0.5, float(ymax) + 0.6))
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
print("\n" + "="*90)
|
||
print("DONE")
|
||
print("="*90)
|
||
|
||
|
||
# Example run:
|
||
plot_confidence_vs_abs_error_with_log("/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration", "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv")
|
||
|
||
|
||
##
|
||
|
||
|
||
# %% Empirical Confidence
|
||
# Empirical stability confidence (from 10 runs) + LLM certainty_percent as secondary signal
|
||
# - Reads all JSONs in a folder (your 10 iterations)
|
||
# - Aggregates by key = unique_id + MedDatum
|
||
# - Computes:
|
||
# * EDSS_mean, EDSS_std, EDSS_iqr, mode/share
|
||
# * empirical_conf_0_100 (based on stability)
|
||
# * llm_conf_mean_0_100 (mean certainty_percent)
|
||
# * combined_conf_0_100 (weighted blend)
|
||
# - Optional: merges GT EDSS and computes abs error on the aggregated prediction
|
||
|
||
import os, glob, json
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
def build_empirical_confidence_table(
|
||
json_dir_path: str,
|
||
ground_truth_path: str | None = None,
|
||
gt_sep: str = ";",
|
||
gt_edss_col: str = "EDSS",
|
||
w_empirical: float = 0.7, # weight for empirical stability
|
||
w_llm: float = 0.3, # weight for LLM self-reported confidence
|
||
tol_mode: float = 0.5, # tolerance to treat EDSS as "same" (EDSS often in 0.5 steps)
|
||
min_runs_expected: int = 10,
|
||
):
|
||
# -----------------------------
|
||
# Helpers
|
||
# -----------------------------
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_number(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
def robust_iqr(x: pd.Series):
|
||
x = x.dropna()
|
||
if len(x) == 0:
|
||
return np.nan
|
||
return float(x.quantile(0.75) - x.quantile(0.25))
|
||
|
||
def stability_to_confidence(std_val: float) -> float:
|
||
"""
|
||
Map EDSS variability across runs to a 0..100 confidence.
|
||
EDSS is typically on 0.5 steps. A natural scale:
|
||
std ~= 0.0 -> ~100
|
||
std ~= 0.25 -> ~75-90
|
||
std ~= 0.5 -> ~50-70
|
||
std >= 1.0 -> low
|
||
Use a smooth exponential mapping.
|
||
"""
|
||
if np.isnan(std_val):
|
||
return np.nan
|
||
# scale parameter: std=0.5 -> exp(-1)=0.367 -> ~36.7
|
||
scale = 0.5
|
||
conf = 100.0 * np.exp(-(std_val / scale))
|
||
# clamp
|
||
return float(np.clip(conf, 0.0, 100.0))
|
||
|
||
def mode_share_with_tolerance(values: np.ndarray, tol: float) -> tuple[float, float]:
|
||
"""
|
||
Compute a 'mode' under tolerance: pick the cluster center (median) and count
|
||
how many values fall within +/- tol. Return (mode_center, share).
|
||
This is robust to tiny float differences.
|
||
"""
|
||
vals = values[~np.isnan(values)]
|
||
if len(vals) == 0:
|
||
return (np.nan, np.nan)
|
||
center = float(np.median(vals))
|
||
share = float(np.mean(np.abs(vals - center) <= tol))
|
||
return (center, share)
|
||
|
||
# -----------------------------
|
||
# Load predictions from all JSONs
|
||
# -----------------------------
|
||
json_files = sorted(glob.glob(os.path.join(json_dir_path, "*.json")))
|
||
if not json_files:
|
||
raise FileNotFoundError(f"No JSON files found in: {json_dir_path}")
|
||
|
||
rows = []
|
||
per_file = []
|
||
total_entries_all = 0
|
||
total_success_all = 0
|
||
skipped_all = {"not_success": 0, "missing_uid_or_date": 0}
|
||
|
||
for fp in json_files:
|
||
with open(fp, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
|
||
total_entries = len(data)
|
||
success_entries = sum(1 for e in data if e.get("success"))
|
||
total_entries_all += total_entries
|
||
total_success_all += success_entries
|
||
|
||
skipped = {"not_success": 0, "missing_uid_or_date": 0}
|
||
loaded = 0
|
||
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
skipped["not_success"] += 1
|
||
continue
|
||
|
||
res = entry.get("result", {})
|
||
uid = res.get("unique_id")
|
||
md = res.get("MedDatum")
|
||
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
skipped["missing_uid_or_date"] += 1
|
||
continue
|
||
|
||
edss = parse_number(res.get("EDSS"))
|
||
conf = parse_number(res.get("certainty_percent"))
|
||
it = res.get("iteration", None)
|
||
|
||
rows.append({
|
||
"file": os.path.basename(fp),
|
||
"iteration": it,
|
||
"unique_id": norm_str(uid),
|
||
"MedDatum": norm_str(md),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": edss,
|
||
"llm_conf": conf,
|
||
})
|
||
loaded += 1
|
||
|
||
skipped_all["not_success"] += skipped["not_success"]
|
||
skipped_all["missing_uid_or_date"] += skipped["missing_uid_or_date"]
|
||
|
||
per_file.append({
|
||
"file": os.path.basename(fp),
|
||
"entries_total": total_entries,
|
||
"entries_success": success_entries,
|
||
"rows_loaded": loaded,
|
||
"skipped_not_success": skipped["not_success"],
|
||
"skipped_missing_uid_or_date": skipped["missing_uid_or_date"],
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
df_file = pd.DataFrame(per_file)
|
||
|
||
# -----------------------------
|
||
# Logs: ingestion
|
||
# -----------------------------
|
||
print("\n" + "="*90)
|
||
print("EMPIRICAL CONFIDENCE (10-RUN STABILITY) + LLM CONFIDENCE (SECONDARY)")
|
||
print("="*90)
|
||
print(f"JSON DIR: {json_dir_path}")
|
||
print(f"JSON files: {len(json_files)}")
|
||
print("\n--- INGEST LOG ---")
|
||
print(f"Total JSON entries: {total_entries_all}")
|
||
print(f"Total success entries:{total_success_all}")
|
||
print(f"Pred rows loaded: {len(df_pred)}")
|
||
print(f"Unique keys in preds: {df_pred['key'].nunique() if len(df_pred) else 0}")
|
||
print(f"Missing EDSS_pred: {df_pred['EDSS_pred'].isna().sum() if len(df_pred) else 0}")
|
||
print(f"Missing llm_conf: {df_pred['llm_conf'].isna().sum() if len(df_pred) else 0}")
|
||
print("Skipped totals:", skipped_all)
|
||
|
||
print("\nPer-file summary (top 10 by name):")
|
||
print(df_file.sort_values("file").head(10))
|
||
|
||
# -----------------------------
|
||
# Aggregate by key (empirical stability)
|
||
# -----------------------------
|
||
if len(df_pred) == 0:
|
||
print("[ERROR] No usable prediction rows.")
|
||
return None
|
||
|
||
# how many runs per key (expect ~10)
|
||
runs_per_key = df_pred.groupby("key")["EDSS_pred"].size().rename("n_rows").reset_index()
|
||
print("\n--- RUNS PER KEY LOG ---")
|
||
print(f"Keys with at least 1 row: {len(runs_per_key)}")
|
||
print("Distribution of rows per key (value_counts):")
|
||
print(runs_per_key["n_rows"].value_counts().sort_index())
|
||
|
||
# Aggregate stats
|
||
def agg_block(g: pd.DataFrame):
|
||
ed = g["EDSS_pred"].to_numpy(dtype=float)
|
||
ll = g["llm_conf"].to_numpy(dtype=float)
|
||
|
||
n_rows = len(g)
|
||
n_edss = int(np.sum(~np.isnan(ed)))
|
||
n_llm = int(np.sum(~np.isnan(ll)))
|
||
|
||
ed_mean = float(np.nanmean(ed)) if n_edss else np.nan
|
||
ed_std = float(np.nanstd(ed, ddof=1)) if n_edss >= 2 else (0.0 if n_edss == 1 else np.nan)
|
||
ed_iqr = robust_iqr(pd.Series(ed))
|
||
mode_center, mode_share = mode_share_with_tolerance(ed, tol=tol_mode)
|
||
|
||
llm_mean = float(np.nanmean(ll)) if n_llm else np.nan
|
||
llm_std = float(np.nanstd(ll, ddof=1)) if n_llm >= 2 else (0.0 if n_llm == 1 else np.nan)
|
||
|
||
emp_conf = stability_to_confidence(ed_std) if not np.isnan(ed_std) else np.nan
|
||
|
||
# Combined confidence (weighted). If one side missing, fall back to the other.
|
||
if np.isnan(emp_conf) and np.isnan(llm_mean):
|
||
comb = np.nan
|
||
elif np.isnan(emp_conf):
|
||
comb = llm_mean
|
||
elif np.isnan(llm_mean):
|
||
comb = emp_conf
|
||
else:
|
||
comb = w_empirical * emp_conf + w_llm * llm_mean
|
||
|
||
return pd.Series({
|
||
"unique_id": g["unique_id"].iloc[0],
|
||
"MedDatum": g["MedDatum"].iloc[0],
|
||
"n_rows": n_rows,
|
||
"n_edss": n_edss,
|
||
"n_llm_conf":n_llm,
|
||
"EDSS_mean": ed_mean,
|
||
"EDSS_std": ed_std,
|
||
"EDSS_iqr": ed_iqr,
|
||
"EDSS_mode_center": mode_center,
|
||
"EDSS_mode_share": mode_share, # fraction within ±tol_mode of median center
|
||
"llm_conf_mean": llm_mean,
|
||
"llm_conf_std": llm_std,
|
||
"empirical_conf_0_100": emp_conf,
|
||
"combined_conf_0_100": float(np.clip(comb, 0.0, 100.0)) if not np.isnan(comb) else np.nan,
|
||
})
|
||
|
||
df_agg = df_pred.groupby("key", as_index=False).apply(agg_block)
|
||
# groupby+apply returns a multiindex sometimes depending on pandas version
|
||
if isinstance(df_agg.index, pd.MultiIndex):
|
||
df_agg = df_agg.reset_index(drop=True)
|
||
|
||
# Logs: aggregation + losses
|
||
print("\n--- AGGREGATION LOG ---")
|
||
print(f"Aggregated keys: {len(df_agg)}")
|
||
print(f"Keys with EDSS in >=1 run: {(df_agg['n_edss'] >= 1).sum()}")
|
||
print(f"Keys with EDSS in >=2 runs (std meaningful): {(df_agg['n_edss'] >= 2).sum()}")
|
||
print(f"Keys missing EDSS in all runs: {(df_agg['n_edss'] == 0).sum()}")
|
||
print(f"Keys missing llm_conf in all runs: {(df_agg['n_llm_conf'] == 0).sum()}")
|
||
|
||
# Expected runs check
|
||
if min_runs_expected is not None:
|
||
print(f"\nKeys with < {min_runs_expected} rows (potential missing iterations):")
|
||
print(df_agg.loc[df_agg["n_rows"] < min_runs_expected, ["key", "n_rows"]].sort_values("n_rows").head(20))
|
||
|
||
# -----------------------------
|
||
# Optional: merge GT and compute error on aggregated EDSS_mean
|
||
# -----------------------------
|
||
if ground_truth_path is not None:
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
need = {"unique_id", "MedDatum", gt_edss_col}
|
||
miss = need - set(df_gt.columns)
|
||
if miss:
|
||
raise ValueError(f"GT missing columns: {miss}. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].apply(parse_number)
|
||
|
||
print("\n--- GT MERGE LOG ---")
|
||
print(f"GT rows: {len(df_gt)} | GT unique keys: {df_gt['key'].nunique()}")
|
||
print(f"GT missing EDSS (numeric): {df_gt['EDSS_gt'].isna().sum()}")
|
||
|
||
df_final = df_agg.merge(df_gt[["key", "EDSS_gt"]], on="key", how="left", validate="one_to_one")
|
||
|
||
print(f"Aggregated keys with GT match: {df_final['EDSS_gt'].notna().sum()} / {len(df_final)}")
|
||
print(f"Aggregated keys missing GT EDSS: {df_final['EDSS_gt'].isna().sum()}")
|
||
|
||
df_final["abs_error_mean"] = (df_final["EDSS_mean"] - df_final["EDSS_gt"]).abs()
|
||
|
||
# How many keys usable for evaluation?
|
||
usable = df_final.dropna(subset=["EDSS_mean", "EDSS_gt"])
|
||
print("\n--- EVAL LOG (AGGREGATED) ---")
|
||
print(f"Keys with both EDSS_mean and EDSS_gt: {len(usable)}")
|
||
if len(usable) > 0:
|
||
print(f"MAE on EDSS_mean vs GT: {usable['abs_error_mean'].mean():.3f}")
|
||
print(f"Median abs error: {usable['abs_error_mean'].median():.3f}")
|
||
|
||
return df_final
|
||
|
||
return df_agg
|
||
|
||
|
||
# Example usage:
|
||
df = build_empirical_confidence_table(json_dir_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration", ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv", w_empirical=0.7, w_llm=0.3, tol_mode=0.5,min_runs_expected=10,)
|
||
df.to_csv("empirical_confidence_table.csv", index=False)
|
||
|
||
##
|
||
|
||
|
||
|
||
|
||
# %% Executive Boxplot
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.patches import Patch
|
||
|
||
def plot_exec_boxplots(df, min_bin_size_warn=10):
|
||
"""
|
||
Two side-by-side boxplots:
|
||
- Left: abs_error_mean grouped by empirical_conf_0_100 quantile bins
|
||
- Right: abs_error_mean grouped by llm_conf_mean quantile bins
|
||
|
||
Adds:
|
||
- Robust qcut labeling (handles ties; bins may be < 4)
|
||
- Data logs + per-bin summary table printed
|
||
- Clear legend explaining each panel and what box elements mean
|
||
"""
|
||
need_cols = ["abs_error_mean", "empirical_conf_0_100", "llm_conf_mean"]
|
||
missing = [c for c in need_cols if c not in df.columns]
|
||
if missing:
|
||
raise ValueError(f"Missing columns in df: {missing}. Available: {df.columns.tolist()}")
|
||
|
||
d = df[need_cols].copy()
|
||
|
||
# -----------------------------
|
||
# Data logs: survivorship
|
||
# -----------------------------
|
||
d_emp = d.dropna(subset=["abs_error_mean", "empirical_conf_0_100"]).copy()
|
||
d_llm = d.dropna(subset=["abs_error_mean", "llm_conf_mean"]).copy()
|
||
|
||
print("\n" + "="*90)
|
||
print("EXECUTIVE BOXPLOTS — DATA LOG + SUMMARY")
|
||
print("="*90)
|
||
print(f"Total rows in df: {len(df)}")
|
||
print(f"Rows for empirical plot: {len(d_emp)} (dropped {len(df) - len(d_emp)})")
|
||
print(f"Rows for LLM plot: {len(d_llm)} (dropped {len(df) - len(d_llm)})")
|
||
|
||
if len(d_emp) == 0 or len(d_llm) == 0:
|
||
print("[ERROR] Not enough data after dropping NaNs to build both plots.")
|
||
return
|
||
|
||
# -----------------------------
|
||
# Robust quantile binning (handles ties)
|
||
# -----------------------------
|
||
# Empirical
|
||
emp_bins = pd.qcut(d_emp["empirical_conf_0_100"], q=4, duplicates="drop")
|
||
k_emp = emp_bins.cat.categories.size
|
||
emp_labels = [f"Q{i+1}" for i in range(k_emp)]
|
||
d_emp["emp_q"] = pd.qcut(d_emp["empirical_conf_0_100"], q=4, duplicates="drop", labels=emp_labels)
|
||
|
||
# LLM
|
||
llm_bins = pd.qcut(d_llm["llm_conf_mean"], q=4, duplicates="drop")
|
||
k_llm = llm_bins.cat.categories.size
|
||
llm_labels = [f"Q{i+1}" for i in range(k_llm)]
|
||
d_llm["llm_q"] = pd.qcut(d_llm["llm_conf_mean"], q=4, duplicates="drop", labels=llm_labels)
|
||
|
||
# Print bin edges (so you can discuss exact thresholds)
|
||
print("\n--- BIN EDGES (actual ranges) ---")
|
||
print("Empirical confidence bins:")
|
||
for i, interval in enumerate(emp_bins.cat.categories):
|
||
print(f" {emp_labels[i]}: {interval}")
|
||
print("LLM confidence bins:")
|
||
for i, interval in enumerate(llm_bins.cat.categories):
|
||
print(f" {llm_labels[i]}: {interval}")
|
||
|
||
# -----------------------------
|
||
# Summary tables (per bin)
|
||
# -----------------------------
|
||
def summarize_bins(df_in, bin_col, conf_col, label):
|
||
g = df_in.groupby(bin_col, observed=True).agg(
|
||
n=("abs_error_mean", "size"),
|
||
mae_mean=("abs_error_mean", "mean"),
|
||
mae_median=("abs_error_mean", "median"),
|
||
mae_q25=("abs_error_mean", lambda x: x.quantile(0.25)),
|
||
mae_q75=("abs_error_mean", lambda x: x.quantile(0.75)),
|
||
conf_mean=(conf_col, "mean"),
|
||
conf_median=(conf_col, "median"),
|
||
).reset_index().rename(columns={bin_col: "bin"})
|
||
g["panel"] = label
|
||
return g[["panel", "bin", "n", "mae_mean", "mae_median", "mae_q25", "mae_q75", "conf_mean", "conf_median"]]
|
||
|
||
summary_emp = summarize_bins(d_emp, "emp_q", "empirical_conf_0_100", "Empirical")
|
||
summary_llm = summarize_bins(d_llm, "llm_q", "llm_conf_mean", "LLM")
|
||
|
||
print("\n--- SUMMARY TABLE: Empirical confidence quartiles (or fewer if ties) ---")
|
||
print(summary_emp.to_string(index=False, float_format=lambda x: f"{x:.3f}"))
|
||
|
||
print("\n--- SUMMARY TABLE: LLM confidence quartiles (or fewer if ties) ---")
|
||
print(summary_llm.to_string(index=False, float_format=lambda x: f"{x:.3f}"))
|
||
|
||
# Warn about small bins
|
||
small_emp = summary_emp.loc[summary_emp["n"] < min_bin_size_warn, ["bin", "n"]]
|
||
small_llm = summary_llm.loc[summary_llm["n"] < min_bin_size_warn, ["bin", "n"]]
|
||
if not small_emp.empty or not small_llm.empty:
|
||
print(f"\n[WARNING] Some bins have < {min_bin_size_warn} points; compare them cautiously.")
|
||
if not small_emp.empty:
|
||
print(" Empirical small bins:")
|
||
print(small_emp.to_string(index=False))
|
||
if not small_llm.empty:
|
||
print(" LLM small bins:")
|
||
print(small_llm.to_string(index=False))
|
||
|
||
# -----------------------------
|
||
# Prepare data for boxplots
|
||
# -----------------------------
|
||
emp_cats = list(d_emp["emp_q"].cat.categories)
|
||
llm_cats = list(d_llm["llm_q"].cat.categories)
|
||
|
||
emp_groups = [d_emp.loc[d_emp["emp_q"] == q, "abs_error_mean"].values for q in emp_cats]
|
||
llm_groups = [d_llm.loc[d_llm["llm_q"] == q, "abs_error_mean"].values for q in llm_cats]
|
||
|
||
# -----------------------------
|
||
# Plot
|
||
# -----------------------------
|
||
fig, axes = plt.subplots(1, 2, figsize=(12, 5), sharey=True)
|
||
|
||
bp0 = axes[0].boxplot(emp_groups, labels=emp_cats, showfliers=False, patch_artist=True)
|
||
bp1 = axes[1].boxplot(llm_groups, labels=llm_cats, showfliers=False, patch_artist=True)
|
||
|
||
# Make panels visually distinct but still simple (no extra clutter)
|
||
for patch in bp0["boxes"]:
|
||
patch.set_alpha(0.6)
|
||
for patch in bp1["boxes"]:
|
||
patch.set_alpha(0.6)
|
||
|
||
axes[0].set_title("Error by Empirical Confidence (quantile bins)")
|
||
axes[0].set_xlabel("Empirical confidence bin")
|
||
axes[0].set_ylabel("Absolute Error (|EDSS_mean − EDSS_gt|)")
|
||
|
||
axes[1].set_title("Error by LLM Confidence (quantile bins)")
|
||
axes[1].set_xlabel("LLM confidence bin")
|
||
|
||
for ax in axes:
|
||
ax.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
# -----------------------------
|
||
# Legend (simple, but useful)
|
||
# -----------------------------
|
||
legend_elements = [
|
||
Patch(facecolor="white", edgecolor="black", label="Box = IQR (25%–75%)"),
|
||
Patch(facecolor="white", edgecolor="black", label="Center line = median"),
|
||
Patch(facecolor="white", edgecolor="black", label="Whiskers = typical range (no outliers shown)"),
|
||
Patch(facecolor="white", edgecolor="white", label="Left panel: empirical stability bins"),
|
||
Patch(facecolor="white", edgecolor="white", label="Right panel: LLM self-reported bins"),
|
||
]
|
||
fig.legend(handles=legend_elements, loc="upper center", ncol=3, frameon=True)
|
||
|
||
plt.tight_layout(rect=[0, 0, 1, 0.90])
|
||
plt.show()
|
||
|
||
print("\n" + "="*90)
|
||
print("DONE")
|
||
print("="*90)
|
||
|
||
|
||
# Example (complete):
|
||
df_final = build_empirical_confidence_table(
|
||
json_dir_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration",
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv",
|
||
w_empirical=0.7,
|
||
w_llm=0.3,
|
||
tol_mode=0.5,
|
||
min_runs_expected=10,
|
||
)
|
||
plot_exec_boxplots(df_final)
|
||
|
||
|
||
##
|
||
|
||
|
||
|
||
# %% Scatter
|
||
|
||
import os, json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
|
||
def scatter_abs_error_by_conf_bins_single_json(
|
||
json_file_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
# ---- Load GT
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ---- Load preds from JSON
|
||
with open(json_file_path, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
|
||
rows = []
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
|
||
rows.append({
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
"confidence": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ---- Merge + filter
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "confidence"]).copy()
|
||
df["abs_error"] = (df["EDSS_pred"] - df["EDSS_gt"]).abs()
|
||
|
||
# ---- Bin confidence into 4 categories
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
df["conf_bin"] = pd.cut(df["confidence"], bins=bins, labels=labels, include_lowest=True)
|
||
df = df.dropna(subset=["conf_bin"]).copy()
|
||
|
||
# ---- Logs
|
||
print("\n--- BIN COUNTS (points plotted) ---")
|
||
print(df["conf_bin"].value_counts().reindex(labels).fillna(0).astype(int))
|
||
print(f"Total points plotted: {len(df)}")
|
||
|
||
# ---- Scatter (categorical x with jitter)
|
||
x_map = {lab: i for i, lab in enumerate(labels)}
|
||
x = df["conf_bin"].map(x_map).astype(float).to_numpy()
|
||
jitter = np.random.uniform(-0.12, 0.12, size=len(df))
|
||
xj = x + jitter
|
||
|
||
plt.figure(figsize=(12, 6))
|
||
plt.scatter(xj, df["abs_error"].to_numpy(), alpha=0.55)
|
||
plt.xticks(range(len(labels)), labels)
|
||
plt.xlabel("certainty_percent category (Iteration 1)")
|
||
plt.ylabel("Absolute Error (|EDSS_pred − EDSS_gt|)")
|
||
plt.title("Absolute Error vs LLM Confidence Category (Single JSON)")
|
||
plt.grid(axis="y", linestyle=":", alpha=0.5)
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
# --- RUN ---
|
||
scatter_abs_error_by_conf_bins_single_json(
|
||
json_file_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json",
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv",
|
||
)
|
||
|
||
##
|
||
|
||
|
||
|
||
|
||
# %% Boxplot2
|
||
|
||
# Boxplot + light jittered points
|
||
# - Single JSON (iteration 1)
|
||
# - X: confidence bin (<70, 70-80, 80-90, 90-100)
|
||
# - Y: absolute error
|
||
# - Legend includes n per bin
|
||
|
||
import json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.patches import Patch
|
||
|
||
def boxplot_with_jitter_abs_error_by_conf_bins_single_json(
|
||
json_file_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
jitter_width=0.12,
|
||
point_alpha=0.25,
|
||
show_outliers=False,
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
# ---- Load GT
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
for col in ["unique_id", "MedDatum", gt_edss_col]:
|
||
if col not in df_gt.columns:
|
||
raise ValueError(f"GT missing column '{col}'. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ---- Load preds from JSON
|
||
with open(json_file_path, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
|
||
rows = []
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
rows.append({
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
"confidence": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ---- Merge + filter
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "confidence"]).copy()
|
||
df["abs_error"] = (df["EDSS_pred"] - df["EDSS_gt"]).abs()
|
||
|
||
# ---- Bin confidence
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
df["conf_bin"] = pd.cut(df["confidence"], bins=bins, labels=labels, include_lowest=True)
|
||
df = df.dropna(subset=["conf_bin"]).copy()
|
||
|
||
# ---- Prepare per-bin arrays
|
||
bin_arrays = [df.loc[df["conf_bin"] == lab, "abs_error"].to_numpy() for lab in labels]
|
||
n_counts = [len(a) for a in bin_arrays]
|
||
|
||
# ---- Plot
|
||
fig, ax = plt.subplots(figsize=(12, 6))
|
||
|
||
# Boxplot (no fliers by default to reduce clutter)
|
||
bp = ax.boxplot(
|
||
bin_arrays,
|
||
labels=labels,
|
||
showfliers=show_outliers,
|
||
patch_artist=True,
|
||
widths=0.55,
|
||
)
|
||
|
||
# Light fill for boxes (no explicit color choices required)
|
||
for b in bp["boxes"]:
|
||
b.set_alpha(0.35)
|
||
|
||
# Jittered points on top
|
||
for i, arr in enumerate(bin_arrays, start=1):
|
||
if len(arr) == 0:
|
||
continue
|
||
x = np.full(len(arr), i, dtype=float)
|
||
x += np.random.uniform(-jitter_width, jitter_width, size=len(arr))
|
||
ax.scatter(x, arr, alpha=point_alpha, s=18)
|
||
|
||
ax.set_title("Absolute Error by LLM Confidence Bin (Iteration 1)")
|
||
ax.set_xlabel("certainty_percent category")
|
||
ax.set_ylabel("Absolute Error (|EDSS_pred − EDSS_gt|)")
|
||
ax.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
# Legend showing n per bin
|
||
legend_handles = [
|
||
Patch(facecolor="white", edgecolor="black", label=f"{lab}: n={n}")
|
||
for lab, n in zip(labels, n_counts)
|
||
]
|
||
ax.legend(handles=legend_handles, title="Bin counts", loc="upper right", frameon=True)
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
# Print counts too (useful for discussion)
|
||
print("\n--- BIN COUNTS (points plotted) ---")
|
||
for lab, n in zip(labels, n_counts):
|
||
print(f"{lab:>18}: n={n}")
|
||
print(f"Total points plotted: {sum(n_counts)}")
|
||
|
||
|
||
# Example run:
|
||
boxplot_with_jitter_abs_error_by_conf_bins_single_json(
|
||
json_file_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json",
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv"
|
||
)
|
||
|
||
##
|
||
|
||
|
||
|
||
|
||
# %% Boxplot3
|
||
|
||
|
||
# Boxplot + jitter with SIGNED error (direction)
|
||
# - Y-axis: signed error = EDSS_pred - EDSS_gt (negative = underestimation, positive = overestimation)
|
||
# - Also prints per-bin summary (n, mean signed error, median, MAE)
|
||
|
||
import json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.patches import Patch
|
||
from matplotlib.lines import Line2D
|
||
|
||
def boxplot_with_jitter_signed_error_by_conf_bins_single_json(
|
||
json_file_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
jitter_width=0.12,
|
||
point_alpha=0.25,
|
||
show_outliers=False,
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
# ---- Load GT
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
for col in ["unique_id", "MedDatum", gt_edss_col]:
|
||
if col not in df_gt.columns:
|
||
raise ValueError(f"GT missing column '{col}'. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ---- Load preds from JSON
|
||
with open(json_file_path, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
|
||
rows = []
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
rows.append({
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
"confidence": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ---- Merge + filter
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "confidence"]).copy()
|
||
|
||
# SIGNED ERROR (direction)
|
||
df["signed_error"] = df["EDSS_pred"] - df["EDSS_gt"]
|
||
df["abs_error"] = df["signed_error"].abs()
|
||
|
||
# ---- Bin confidence
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
df["conf_bin"] = pd.cut(df["confidence"], bins=bins, labels=labels, include_lowest=True)
|
||
df = df.dropna(subset=["conf_bin"]).copy()
|
||
|
||
# ---- Prepare arrays
|
||
bin_arrays = [df.loc[df["conf_bin"] == lab, "signed_error"].to_numpy() for lab in labels]
|
||
n_counts = [len(a) for a in bin_arrays]
|
||
|
||
# ---- Plot
|
||
fig, ax = plt.subplots(figsize=(12, 6))
|
||
|
||
bp = ax.boxplot(
|
||
bin_arrays,
|
||
labels=labels,
|
||
showfliers=show_outliers,
|
||
patch_artist=True,
|
||
widths=0.55,
|
||
)
|
||
|
||
for b in bp["boxes"]:
|
||
b.set_alpha(0.35)
|
||
|
||
# Jittered points
|
||
for i, arr in enumerate(bin_arrays, start=1):
|
||
if len(arr) == 0:
|
||
continue
|
||
x = np.full(len(arr), i, dtype=float)
|
||
x += np.random.uniform(-jitter_width, jitter_width, size=len(arr))
|
||
ax.scatter(x, arr, alpha=point_alpha, s=18)
|
||
|
||
# Zero line to show over/under clearly
|
||
ax.axhline(0, linewidth=1.5, linestyle="--")
|
||
|
||
ax.set_title("Signed Error by LLM Confidence Bin (Iteration 1)")
|
||
ax.set_xlabel("certainty_percent category")
|
||
ax.set_ylabel("Signed Error (EDSS_pred − EDSS_gt)")
|
||
ax.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
# Legend with n per bin + zero-line meaning
|
||
legend_handles = [
|
||
Patch(facecolor="white", edgecolor="black", label=f"{lab}: n={n}")
|
||
for lab, n in zip(labels, n_counts)
|
||
]
|
||
legend_handles.append(Line2D([0], [0], linestyle="--", color="black", label="0 = unbiased (over/under split)"))
|
||
ax.legend(handles=legend_handles, title="Bin counts", loc="upper right", frameon=True)
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
# ---- Print per-bin summary to discuss
|
||
print("\n--- PER-BIN SUMMARY (points plotted) ---")
|
||
for lab in labels:
|
||
sub = df.loc[df["conf_bin"] == lab]
|
||
n = len(sub)
|
||
if n == 0:
|
||
print(f"{lab:>18}: n=0")
|
||
continue
|
||
print(
|
||
f"{lab:>18}: n={n:3d} | "
|
||
f"mean signed={sub['signed_error'].mean(): .3f} | "
|
||
f"median signed={sub['signed_error'].median(): .3f} | "
|
||
f"MAE={sub['abs_error'].mean(): .3f}"
|
||
)
|
||
print(f"Total points plotted: {len(df)}")
|
||
|
||
|
||
# Example run:
|
||
boxplot_with_jitter_signed_error_by_conf_bins_single_json(
|
||
json_file_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json",
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv"
|
||
)
|
||
##
|
||
|
||
|
||
|
||
# %% jitter and violin 10x10
|
||
|
||
# Violin + jitter (all JSONs in folder), with signed error
|
||
# - X: confidence bins (<70, 70-80, 80-90, 90-100)
|
||
# - Y: signed error = EDSS_pred - EDSS_gt (direction)
|
||
# - Prints bin counts (n) and puts n into the legend
|
||
|
||
import os, glob, json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
from matplotlib.patches import Patch
|
||
from matplotlib.lines import Line2D
|
||
|
||
def violin_jitter_signed_error_all_jsons(
|
||
json_dir_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
jitter_width=0.12,
|
||
point_alpha=0.20,
|
||
point_size=10,
|
||
violin_inner="quartile", # 'quartile', 'box', 'stick', or None
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
# ---- Load GT
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
for col in ["unique_id", "MedDatum", gt_edss_col]:
|
||
if col not in df_gt.columns:
|
||
raise ValueError(f"GT missing column '{col}'. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ---- Load preds from ALL JSONs
|
||
json_files = sorted(glob.glob(os.path.join(json_dir_path, "*.json")))
|
||
if not json_files:
|
||
raise FileNotFoundError(f"No JSON files found in: {json_dir_path}")
|
||
|
||
rows = []
|
||
for fp in json_files:
|
||
with open(fp, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
rows.append({
|
||
"file": os.path.basename(fp),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
"confidence": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ---- Merge + filter
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "confidence"]).copy()
|
||
df["signed_error"] = df["EDSS_pred"] - df["EDSS_gt"]
|
||
|
||
# ---- Bin confidence
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
df["conf_bin"] = pd.cut(df["confidence"], bins=bins, labels=labels, include_lowest=True)
|
||
df = df.dropna(subset=["conf_bin"]).copy()
|
||
|
||
# ---- Counts + log
|
||
counts = df["conf_bin"].value_counts().reindex(labels).fillna(0).astype(int)
|
||
print("\n--- BIN COUNTS (all JSONs) ---")
|
||
for lab in labels:
|
||
print(f"{lab:>18}: n={counts[lab]}")
|
||
print(f"Total points plotted: {len(df)}")
|
||
print(f"JSON files: {len(json_files)}")
|
||
|
||
# Ensure ordering for seaborn
|
||
df["conf_bin"] = pd.Categorical(df["conf_bin"], categories=labels, ordered=True)
|
||
|
||
# ---- Plot
|
||
plt.figure(figsize=(12, 6))
|
||
|
||
# Violin (density)
|
||
sns.violinplot(
|
||
data=df,
|
||
x="conf_bin",
|
||
y="signed_error",
|
||
order=labels,
|
||
inner=violin_inner,
|
||
cut=0
|
||
)
|
||
|
||
# Jittered points (manual jitter to keep it consistent and fast)
|
||
x_map = {lab: i for i, lab in enumerate(labels)}
|
||
x = df["conf_bin"].map(x_map).astype(float).to_numpy()
|
||
xj = x + np.random.uniform(-jitter_width, jitter_width, size=len(df))
|
||
plt.scatter(xj, df["signed_error"].to_numpy(), alpha=point_alpha, s=point_size)
|
||
|
||
# Zero line (over/under split)
|
||
plt.axhline(0, linestyle="--", linewidth=1.5)
|
||
|
||
plt.xticks(range(len(labels)), labels)
|
||
plt.xlabel("certainty_percent category (all iterations)")
|
||
plt.ylabel("Signed Error (EDSS_pred − EDSS_gt)")
|
||
plt.title("Signed Error vs LLM Confidence Category — Violin + Jitter (All JSONs)")
|
||
plt.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
# Legend with n per bin
|
||
legend_handles = [
|
||
Patch(facecolor="white", edgecolor="black", label=f"{lab}: n={int(counts[lab])}")
|
||
for lab in labels
|
||
]
|
||
legend_handles.append(Line2D([0], [0], linestyle="--", color="black", label="0 = unbiased (over/under split)"))
|
||
plt.legend(handles=legend_handles, title="Bin counts", loc="upper right", frameon=True)
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
|
||
# Example run:
|
||
violin_jitter_signed_error_all_jsons(
|
||
json_dir_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration",
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv"
|
||
)
|
||
|
||
##
|
||
|
||
|
||
|
||
# %% jitter and violin 10x1
|
||
|
||
|
||
# Adjusted: Violin + jitter (ALL JSONs for points) but X-bins come ONLY from JSON #1 (reference)
|
||
# Fixes:
|
||
# 1) Legend has colors matching bins
|
||
# 2) Legend placed OUTSIDE plot area
|
||
# 3) X-axis binning uses certainty_percent from JSON1 (by key), then all iterations' points inherit that bin
|
||
|
||
import os, glob, json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
from matplotlib.patches import Patch
|
||
from matplotlib.lines import Line2D
|
||
|
||
def violin_jitter_signed_error_all_jsons_xbins_from_json1(
|
||
json_dir_path,
|
||
json1_file_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
jitter_width=0.12,
|
||
point_alpha=0.18,
|
||
point_size=10,
|
||
violin_inner="quartile", # 'quartile', 'box', 'stick', or None
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
# ----------------------------
|
||
# Load GT
|
||
# ----------------------------
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
for col in ["unique_id", "MedDatum", gt_edss_col]:
|
||
if col not in df_gt.columns:
|
||
raise ValueError(f"GT missing column '{col}'. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ----------------------------
|
||
# Load JSON1 and build reference bins by KEY
|
||
# ----------------------------
|
||
with open(json1_file_path, "r", encoding="utf-8") as f:
|
||
data1 = json.load(f)
|
||
|
||
ref_rows = []
|
||
for entry in data1:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
ref_rows.append({
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"confidence_ref": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_ref = pd.DataFrame(ref_rows)
|
||
|
||
# If JSON1 has duplicates for a key (unlikely, but safe), take the first non-null confidence
|
||
df_ref = (df_ref.sort_values("confidence_ref")
|
||
.groupby("key", as_index=False)["confidence_ref"]
|
||
.apply(lambda s: s.dropna().iloc[0] if s.dropna().any() else np.nan))
|
||
if isinstance(df_ref.index, pd.MultiIndex):
|
||
df_ref = df_ref.reset_index(drop=True)
|
||
|
||
# Confidence bins
|
||
bins = [0, 70, 80, 90, 100]
|
||
labels = ["Low (<70%)", "Moderate (70-80%)", "High (80-90%)", "Very High (90-100%)"]
|
||
df_ref["conf_bin_ref"] = pd.cut(df_ref["confidence_ref"], bins=bins, labels=labels, include_lowest=True)
|
||
df_ref = df_ref.dropna(subset=["conf_bin_ref"]).copy()
|
||
|
||
# ----------------------------
|
||
# Load ALL JSONs (all points)
|
||
# ----------------------------
|
||
json_files = sorted(glob.glob(os.path.join(json_dir_path, "*.json")))
|
||
if not json_files:
|
||
raise FileNotFoundError(f"No JSON files found in: {json_dir_path}")
|
||
|
||
rows = []
|
||
for fp in json_files:
|
||
with open(fp, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
rows.append({
|
||
"file": os.path.basename(fp),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ----------------------------
|
||
# Merge: preds + GT + reference bins (from JSON1)
|
||
# ----------------------------
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.merge(df_ref[["key", "conf_bin_ref"]], on="key", how="inner", validate="many_to_one")
|
||
|
||
# filter for plotting
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "conf_bin_ref"]).copy()
|
||
df["signed_error"] = df["EDSS_pred"] - df["EDSS_gt"]
|
||
|
||
# ordering
|
||
df["conf_bin_ref"] = pd.Categorical(df["conf_bin_ref"], categories=labels, ordered=True)
|
||
|
||
# ----------------------------
|
||
# Logs + counts
|
||
# ----------------------------
|
||
counts = df["conf_bin_ref"].value_counts().reindex(labels).fillna(0).astype(int)
|
||
|
||
print("\n--- BIN COUNTS (ALL JSON points, binned by JSON1 confidence) ---")
|
||
for lab in labels:
|
||
print(f"{lab:>18}: n={int(counts[lab])}")
|
||
print(f"Total points plotted: {len(df)}")
|
||
print(f"JSON files used for points: {len(json_files)}")
|
||
print(f"Reference JSON1 bins derived from: {os.path.basename(json1_file_path)}")
|
||
print(f"Keys in reference (after binning & non-null): {df_ref['key'].nunique()}")
|
||
|
||
# ----------------------------
|
||
# Colors + legend patches
|
||
# ----------------------------
|
||
palette = sns.color_palette("Blues", n_colors=len(labels))
|
||
bin_colors = {lab: palette[i] for i, lab in enumerate(labels)}
|
||
|
||
legend_handles = [
|
||
Patch(facecolor=bin_colors[lab], edgecolor="black", label=f"{lab}: n={int(counts[lab])}")
|
||
for lab in labels
|
||
]
|
||
legend_handles.append(Line2D([0], [0], linestyle="--", color="black", label="0 = unbiased (over/under split)"))
|
||
|
||
# ----------------------------
|
||
# Plot (legend outside)
|
||
# ----------------------------
|
||
fig, ax = plt.subplots(figsize=(12.5, 6))
|
||
|
||
sns.violinplot(
|
||
data=df,
|
||
x="conf_bin_ref",
|
||
y="signed_error",
|
||
order=labels,
|
||
inner=violin_inner,
|
||
cut=0,
|
||
palette=[bin_colors[l] for l in labels],
|
||
ax=ax,
|
||
)
|
||
|
||
# jittered points (manual jitter)
|
||
x_map = {lab: i for i, lab in enumerate(labels)}
|
||
x = df["conf_bin_ref"].map(x_map).astype(float).to_numpy()
|
||
xj = x + np.random.uniform(-jitter_width, jitter_width, size=len(df))
|
||
ax.scatter(xj, df["signed_error"].to_numpy(), alpha=point_alpha, s=point_size)
|
||
|
||
ax.axhline(0, linestyle="--", linewidth=1.5)
|
||
|
||
ax.set_xlabel("certainty_percent category (from JSON 1 as reference)")
|
||
ax.set_ylabel("Signed Error (EDSS_pred − EDSS_gt)")
|
||
ax.set_title("Signed Error vs LLM Confidence Category — Violin + Jitter (All JSONs)\nBinned by JSON 1 certainty_percent")
|
||
ax.grid(axis="y", linestyle=":", alpha=0.5)
|
||
|
||
# Legend outside (right)
|
||
ax.legend(
|
||
handles=legend_handles,
|
||
title="Bin counts",
|
||
loc="center left",
|
||
bbox_to_anchor=(1.02, 0.5),
|
||
frameon=True
|
||
)
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
|
||
# Example run:
|
||
json1_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json"
|
||
violin_jitter_signed_error_all_jsons_xbins_from_json1(
|
||
json_dir_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration",
|
||
json1_file_path=json1_path,
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv")
|
||
|
||
##
|
||
|
||
|
||
# %% Coorelation
|
||
|
||
# Correlation plot (RAW certainty_percent) vs error
|
||
# - Uses ALL JSONs as points
|
||
# - Uses JSON1 certainty_percent as the x-value reference (per key)
|
||
# - Y can be abs_error or signed_error (choose with y_mode)
|
||
# - Prints Spearman + Pearson correlations
|
||
# - Adds a simple linear trend line
|
||
|
||
import os, glob, json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
|
||
def correlation_scatter_raw_certainty_json1_reference(
|
||
json_dir_path,
|
||
json1_file_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
y_mode="abs", # "abs" or "signed"
|
||
point_alpha=0.18,
|
||
point_size=12,
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
def rankdata(a):
|
||
# Average-rank for ties (Spearman needs ranks)
|
||
s = pd.Series(a)
|
||
return s.rank(method="average").to_numpy()
|
||
|
||
# ----------------------------
|
||
# Load GT
|
||
# ----------------------------
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
for col in ["unique_id", "MedDatum", gt_edss_col]:
|
||
if col not in df_gt.columns:
|
||
raise ValueError(f"GT missing column '{col}'. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ----------------------------
|
||
# Load JSON1 reference certainty_percent (per key)
|
||
# ----------------------------
|
||
with open(json1_file_path, "r", encoding="utf-8") as f:
|
||
data1 = json.load(f)
|
||
|
||
ref_rows = []
|
||
for entry in data1:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
ref_rows.append({
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"certainty_ref": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_ref = pd.DataFrame(ref_rows)
|
||
|
||
# Deduplicate keys if needed: take first non-null certainty
|
||
df_ref = (df_ref.dropna(subset=["certainty_ref"])
|
||
.groupby("key", as_index=False)["certainty_ref"]
|
||
.first())
|
||
|
||
# ----------------------------
|
||
# Load ALL JSON predictions (points)
|
||
# ----------------------------
|
||
json_files = sorted(glob.glob(os.path.join(json_dir_path, "*.json")))
|
||
if not json_files:
|
||
raise FileNotFoundError(f"No JSON files found in: {json_dir_path}")
|
||
|
||
rows = []
|
||
for fp in json_files:
|
||
with open(fp, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
rows.append({
|
||
"file": os.path.basename(fp),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ----------------------------
|
||
# Merge: preds + GT + JSON1 reference certainty
|
||
# ----------------------------
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.merge(df_ref[["key", "certainty_ref"]], on="key", how="inner", validate="many_to_one")
|
||
|
||
# Filter needed fields
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "certainty_ref"]).copy()
|
||
|
||
df["signed_error"] = df["EDSS_pred"] - df["EDSS_gt"]
|
||
df["abs_error"] = df["signed_error"].abs()
|
||
y_col = "abs_error" if y_mode == "abs" else "signed_error"
|
||
|
||
# ----------------------------
|
||
# Logs
|
||
# ----------------------------
|
||
print("\n" + "="*90)
|
||
print("CORRELATION: RAW certainty_percent (JSON1 reference) vs ERROR (ALL JSON points)")
|
||
print("="*90)
|
||
print(f"JSON DIR (points): {json_dir_path} | files: {len(json_files)}")
|
||
print(f"JSON1 reference: {os.path.basename(json1_file_path)}")
|
||
print(f"Points available after merge+filter: {len(df)}")
|
||
print(f"Unique keys in plot: {df['key'].nunique()}")
|
||
print(f"Y mode: {y_mode} ({y_col})")
|
||
|
||
# ----------------------------
|
||
# Correlations (Pearson + Spearman)
|
||
# ----------------------------
|
||
x = df["certainty_ref"].to_numpy(dtype=float)
|
||
y = df[y_col].to_numpy(dtype=float)
|
||
|
||
# Pearson
|
||
pearson = np.corrcoef(x, y)[0, 1] if len(df) >= 2 else np.nan
|
||
|
||
# Spearman = Pearson corr of ranks
|
||
rx = rankdata(x)
|
||
ry = rankdata(y)
|
||
spearman = np.corrcoef(rx, ry)[0, 1] if len(df) >= 2 else np.nan
|
||
|
||
print(f"\nPearson r: {pearson:.4f}")
|
||
print(f"Spearman ρ: {spearman:.4f}")
|
||
|
||
# ----------------------------
|
||
# Trend line (simple linear fit)
|
||
# ----------------------------
|
||
# Fit y = a*x + b
|
||
if len(df) >= 2:
|
||
a, b = np.polyfit(x, y, 1)
|
||
else:
|
||
a, b = np.nan, np.nan
|
||
|
||
# ----------------------------
|
||
# Plot
|
||
# ----------------------------
|
||
plt.figure(figsize=(12, 6))
|
||
plt.scatter(x, y, alpha=point_alpha, s=point_size)
|
||
|
||
# trend line across full x-range
|
||
if np.isfinite(a) and np.isfinite(b):
|
||
xs = np.linspace(np.nanmin(x), np.nanmax(x), 200)
|
||
plt.plot(xs, a * xs + b, linestyle="--", linewidth=2)
|
||
|
||
plt.xlabel("certainty_percent (from JSON 1, per key)")
|
||
ylabel = "Absolute Error |EDSS_pred − EDSS_gt|" if y_mode == "abs" else "Signed Error (EDSS_pred − EDSS_gt)"
|
||
plt.ylabel(ylabel)
|
||
plt.title(f"Correlation of JSON1 certainty_percent vs {y_col} (All iterations)\n"
|
||
f"Pearson r={pearson:.3f} | Spearman ρ={spearman:.3f}")
|
||
plt.grid(linestyle=":", alpha=0.5)
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
|
||
# Example run:
|
||
json1_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json"
|
||
correlation_scatter_raw_certainty_json1_reference(
|
||
json_dir_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration",
|
||
json1_file_path=json1_path,
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv",
|
||
y_mode="abs" # or "signed"
|
||
)
|
||
##
|
||
|
||
# %% Correlation adjusted
|
||
|
||
# Correlation scatter (RAW certainty_percent from JSON1) vs error (all JSON points)
|
||
# Adds:
|
||
# 1) Legend (points, trend line) + Pearson/Spearman shown in legend and title
|
||
# 2) Trend line color set to high-contrast (black by default)
|
||
# 3) Density coloring: dots colored by local point density (bluer = more cases) + colorbar
|
||
|
||
import os, glob, json
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.lines import Line2D
|
||
from matplotlib.colors import LogNorm, PowerNorm
|
||
|
||
def correlation_scatter_raw_certainty_json1_reference(
|
||
json_dir_path,
|
||
json1_file_path,
|
||
ground_truth_path,
|
||
gt_sep=";",
|
||
gt_edss_col="EDSS",
|
||
y_mode="abs", # "abs" or "signed"
|
||
point_alpha=0.85, # higher alpha works better with density coloring
|
||
point_size=14,
|
||
trend_color="black", # high-contrast line
|
||
save_svg_path=None,
|
||
dpi=300
|
||
):
|
||
def norm_str(x):
|
||
return str(x).strip().lower()
|
||
|
||
def parse_edss(x):
|
||
if x is None:
|
||
return np.nan
|
||
s = str(x).strip()
|
||
if s == "" or s.lower() in {"nan", "none", "null"}:
|
||
return np.nan
|
||
s = s.replace(",", ".")
|
||
return pd.to_numeric(s, errors="coerce")
|
||
|
||
def rankdata(a):
|
||
return pd.Series(a).rank(method="average").to_numpy()
|
||
|
||
# ----------------------------
|
||
# Load GT
|
||
# ----------------------------
|
||
df_gt = pd.read_csv(ground_truth_path, sep=gt_sep)
|
||
for col in ["unique_id", "MedDatum", gt_edss_col]:
|
||
if col not in df_gt.columns:
|
||
raise ValueError(f"GT missing column '{col}'. Available: {df_gt.columns.tolist()}")
|
||
|
||
df_gt["unique_id"] = df_gt["unique_id"].map(norm_str)
|
||
df_gt["MedDatum"] = df_gt["MedDatum"].map(norm_str)
|
||
df_gt["key"] = df_gt["unique_id"] + "_" + df_gt["MedDatum"]
|
||
df_gt["EDSS_gt"] = df_gt[gt_edss_col].map(parse_edss)
|
||
|
||
# ----------------------------
|
||
# Load JSON1 reference certainty_percent (per key)
|
||
# ----------------------------
|
||
with open(json1_file_path, "r", encoding="utf-8") as f:
|
||
data1 = json.load(f)
|
||
|
||
ref_rows = []
|
||
for entry in data1:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
ref_rows.append({
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"certainty_ref": pd.to_numeric(res.get("certainty_percent"), errors="coerce"),
|
||
})
|
||
|
||
df_ref = pd.DataFrame(ref_rows)
|
||
df_ref = (df_ref.dropna(subset=["certainty_ref"])
|
||
.groupby("key", as_index=False)["certainty_ref"]
|
||
.first())
|
||
|
||
# ----------------------------
|
||
# Load ALL JSON predictions (points)
|
||
# ----------------------------
|
||
json_files = sorted(glob.glob(os.path.join(json_dir_path, "*.json")))
|
||
if not json_files:
|
||
raise FileNotFoundError(f"No JSON files found in: {json_dir_path}")
|
||
|
||
rows = []
|
||
for fp in json_files:
|
||
with open(fp, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
for entry in data:
|
||
if not entry.get("success"):
|
||
continue
|
||
res = entry.get("result", {})
|
||
uid, md = res.get("unique_id"), res.get("MedDatum")
|
||
if uid is None or md is None or str(uid).strip() == "" or str(md).strip() == "":
|
||
continue
|
||
rows.append({
|
||
"file": os.path.basename(fp),
|
||
"key": norm_str(uid) + "_" + norm_str(md),
|
||
"EDSS_pred": parse_edss(res.get("EDSS")),
|
||
})
|
||
|
||
df_pred = pd.DataFrame(rows)
|
||
|
||
# ----------------------------
|
||
# Merge: preds + GT + JSON1 reference certainty
|
||
# ----------------------------
|
||
df = df_pred.merge(df_gt[["key", "EDSS_gt"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.merge(df_ref[["key", "certainty_ref"]], on="key", how="inner", validate="many_to_one")
|
||
df = df.dropna(subset=["EDSS_gt", "EDSS_pred", "certainty_ref"]).copy()
|
||
|
||
df["signed_error"] = df["EDSS_pred"] - df["EDSS_gt"]
|
||
df["abs_error"] = df["signed_error"].abs()
|
||
y_col = "abs_error" if y_mode == "abs" else "signed_error"
|
||
|
||
# ----------------------------
|
||
# Correlations
|
||
# ----------------------------
|
||
x = df["certainty_ref"].to_numpy(dtype=float)
|
||
y = df[y_col].to_numpy(dtype=float)
|
||
|
||
pearson = np.corrcoef(x, y)[0, 1] if len(df) >= 2 else np.nan
|
||
rx, ry = rankdata(x), rankdata(y)
|
||
spearman = np.corrcoef(rx, ry)[0, 1] if len(df) >= 2 else np.nan
|
||
|
||
# ----------------------------
|
||
# Trend line (linear fit)
|
||
# ----------------------------
|
||
if len(df) >= 2:
|
||
a, b = np.polyfit(x, y, 1)
|
||
else:
|
||
a, b = np.nan, np.nan
|
||
|
||
# ----------------------------
|
||
# Density coloring (2D histogram bin counts)
|
||
# "how blue" = how many points are around that location
|
||
# ----------------------------
|
||
# Choose binning resolution (balanced for ~thousands of points)
|
||
x_bins = 50
|
||
y_bins = 50
|
||
|
||
# Compute bin index per point
|
||
x_edges = np.linspace(np.nanmin(x), np.nanmax(x), x_bins + 1)
|
||
y_edges = np.linspace(np.nanmin(y), np.nanmax(y), y_bins + 1)
|
||
|
||
xi = np.clip(np.digitize(x, x_edges) - 1, 0, x_bins - 1)
|
||
yi = np.clip(np.digitize(y, y_edges) - 1, 0, y_bins - 1)
|
||
|
||
# 2D counts
|
||
counts2d = np.zeros((x_bins, y_bins), dtype=int)
|
||
for i in range(len(x)):
|
||
counts2d[xi[i], yi[i]] += 1
|
||
|
||
# density per point = count of its bin
|
||
density = np.array([counts2d[xi[i], yi[i]] for i in range(len(x))], dtype=float)
|
||
|
||
# Plot low density first, high density last (so dense points are visible)
|
||
order = np.argsort(density)
|
||
x_o, y_o, d_o = x[order], y[order], density[order]
|
||
|
||
|
||
|
||
|
||
# ... keep everything above the "Plot" section identical ...
|
||
|
||
# ----------------------------
|
||
# Plot (IMPROVED COLORS)
|
||
# ----------------------------
|
||
fig, ax = plt.subplots(figsize=(12.5, 6))
|
||
|
||
# Option A (recommended): logarithmic color scaling
|
||
# Add +1 to avoid log(0)
|
||
d_plot = d_o + 1
|
||
|
||
# clip vmax so one extreme bin doesn't wash everything out
|
||
vmax = np.percentile(d_plot, 99) # try 95 or 99 depending on your data
|
||
norm = LogNorm(vmin=1, vmax=max(2, vmax))
|
||
|
||
sc = ax.scatter(
|
||
x_o, y_o,
|
||
c=d_plot,
|
||
cmap="Blues",
|
||
norm=norm,
|
||
s=point_size,
|
||
alpha=point_alpha,
|
||
linewidths=0
|
||
)
|
||
|
||
# Trend line (black)
|
||
if np.isfinite(a) and np.isfinite(b):
|
||
xs = np.linspace(np.nanmin(x), np.nanmax(x), 200)
|
||
ax.plot(xs, a * xs + b, linestyle="--", linewidth=2.5, color=trend_color)
|
||
|
||
ax.set_xlabel("certainty percent")
|
||
ax.set_ylabel("Absolute Error" if y_mode == "abs" else "Signed Error (EDSS_pred − EDSS_gt)")
|
||
# ax.set_title(
|
||
# f"Correlation: JSON1 certainty_percent vs {y_col} (All iterations)\n"
|
||
# f"Pearson r={pearson:.3f} | Spearman ρ={spearman:.3f}"
|
||
# )
|
||
ax.grid(linestyle=":", alpha=0.5)
|
||
|
||
# Colorbar
|
||
cbar = plt.colorbar(sc, ax=ax)
|
||
cbar.set_label("Local density (count of cases in bin, log-scaled)")
|
||
|
||
# Legend
|
||
legend_items = [
|
||
Line2D([0], [0], marker="o", linestyle="None", color="navy",
|
||
label=f"Data points (n={len(df)})"),
|
||
Line2D([0], [0], linestyle="--", color=trend_color, linewidth=2.5,
|
||
label=f"Linear trend (Pearson r={pearson:.3f})"),
|
||
]
|
||
ax.legend(handles=legend_items, loc="upper right", frameon=True, title="Legend")
|
||
|
||
plt.tight_layout()
|
||
# Save as SVG (optional)
|
||
if save_svg_path:
|
||
fig.savefig(save_svg_path, format="svg", bbox_inches="tight", dpi=dpi)
|
||
print(f"[SAVED] {save_svg_path}")
|
||
|
||
plt.show()
|
||
|
||
json1_path = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique_results_iter_1_20260212_020628.json"
|
||
|
||
correlation_scatter_raw_certainty_json1_reference(
|
||
json_dir_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration",
|
||
json1_file_path=json1_path,
|
||
ground_truth_path="/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv",
|
||
y_mode="abs",
|
||
# save_svg_path="/home/shahin/Lab/Doktorarbeit/Barcelona/results/corr_json1_abs_error.svg"
|
||
)
|
||
|
||
##
|
||
|
||
|
||
|