diff --git a/Data/certainty_show.py b/Data/certainty_show.py new file mode 100644 index 0000000..a301379 --- /dev/null +++ b/Data/certainty_show.py @@ -0,0 +1,687 @@ +# %% Explore Dist Plot +import pandas as pd +import json +import glob +import os +import re +import matplotlib.pyplot as plt + +def plot_edss_distribution_per_iteration(json_dir_path): + # 1. Reuse your categorization logic + def categorize_edss(value): + if pd.isna(value): return 'Unknown' + elif value <= 1.0: return '0-1' + elif value <= 2.0: return '1-2' + elif value <= 3.0: return '2-3' + elif value <= 4.0: return '3-4' + elif value <= 5.0: return '4-5' + elif value <= 6.0: return '5-6' + elif value <= 7.0: return '6-7' + elif value <= 8.0: return '7-8' + elif value <= 9.0: return '8-9' + elif value <= 10.0: return '9-10' + else: return '10+' + + # 2. Extract data from all files with Numerical Sorting + all_records = [] + json_files = glob.glob(os.path.join(json_dir_path, "*.json")) + + # Natural sort function to handle Iter 1, Iter 2 ... Iter 10 + def natural_key(string_): + return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] + + json_files.sort(key=natural_key) + + for i, file_path in enumerate(json_files): + # We use the index + 1 for the label to ensure Iter 1 to Iter 10 order + iter_label = f"Iter {i+1}" + with open(file_path, 'r', encoding='utf-8') as f: + try: + data = json.load(f) + for entry in data: + if entry.get("success"): + val = entry["result"].get("EDSS") + all_records.append({ + 'Iteration': iter_label, + 'Category': categorize_edss(val), + 'Order': i # Used to maintain sort order in the table + }) + except Exception as e: + print(f"Error reading {file_path}: {e}") + + df = pd.DataFrame(all_records) + + # 3. Create a Frequency Table (Crosstab) + # Pivot so iterations are on the X-axis + dist_table = pd.crosstab(df['Iteration'], df['Category']) + + # Ensure the rows (Iterations) stay in the 1-10 order + iter_order = [f"Iter {i+1}" for i in range(len(json_files))] + dist_table = dist_table.reindex(iter_order) + + # Ensure columns follow clinical order + fixed_labels = ['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'] + available_labels = [l for l in fixed_labels if l in dist_table.columns] + dist_table = dist_table[available_labels] + + # 4. Plotting + ax = dist_table.plot(kind='bar', stacked=True, figsize=(14, 8), colormap='viridis', edgecolor='white') + + plt.title('Distribution of Predicted EDSS Categories per Iteration', fontsize=15, pad=20) + plt.xlabel('JSON Iteration File', fontsize=12) + plt.ylabel('Number of Cases (Count)', fontsize=12) + plt.xticks(rotation=0) + + # Move legend outside to the right + plt.legend(title="EDSS Category", bbox_to_anchor=(1.05, 1), loc='upper left') + + # Add total count labels on top of bars + for i, (name, row) in enumerate(dist_table.iterrows()): + total = row.sum() + if total > 0: + plt.text(i, total + 2, f'Total: {int(total)}', ha='center', va='bottom', fontweight='bold') + + plt.tight_layout() + plt.show() + + return dist_table +# Usage: +counts_table = plot_edss_distribution_per_iteration('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration') +print(counts_table) +## + + +# %% Explore Table +import pandas as pd +import json +import glob +import os +import re + +def generate_edss_distribution_csv(json_dir_path, output_filename='edss_distribution_summary.csv'): + # 1. Categorization logic + def categorize_edss(value): + if pd.isna(value): return 'Unknown' + elif value <= 1.0: return '0-1' + elif value <= 2.0: return '1-2' + elif value <= 3.0: return '2-3' + elif value <= 4.0: return '3-4' + elif value <= 5.0: return '4-5' + elif value <= 6.0: return '5-6' + elif value <= 7.0: return '6-7' + elif value <= 8.0: return '7-8' + elif value <= 9.0: return '8-9' + elif value <= 10.0: return '9-10' + else: return '10+' + + # 2. Extract data from files with Natural Sorting + all_records = [] + json_files = glob.glob(os.path.join(json_dir_path, "*.json")) + + def natural_key(string_): + return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] + + json_files.sort(key=natural_key) + + for i, file_path in enumerate(json_files): + iter_label = f"Iter {i+1}" + with open(file_path, 'r', encoding='utf-8') as f: + try: + data = json.load(f) + for entry in data: + if entry.get("success"): + val = entry["result"].get("EDSS") + all_records.append({ + 'Iteration': iter_label, + 'Category': categorize_edss(val) + }) + except Exception as e: + print(f"Error reading {file_path}: {e}") + + df = pd.DataFrame(all_records) + + # 3. Create Frequency Table (Crosstab) + dist_table = pd.crosstab(df['Iteration'], df['Category']) + + # 4. Reindex Rows (Numerical order) and Columns (Clinical order) + iter_order = [f"Iter {i+1}" for i in range(len(json_files))] + dist_table = dist_table.reindex(iter_order) + + fixed_labels = ['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'] + available_labels = [l for l in fixed_labels if l in dist_table.columns] + dist_table = dist_table[available_labels] + + # Fill missing categories with 0 and convert to integers + dist_table = dist_table.fillna(0).astype(int) + + # 5. Add "Total" row at the end + # This sums the counts for each category across all iterations + dist_table.loc['Total Sum'] = dist_table.sum() + + # 6. Save to CSV + dist_table.to_csv(output_filename) + print(f"Table successfully saved to: {output_filename}") + + return dist_table + +# Usage: +final_table = generate_edss_distribution_csv('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration') +print(final_table) +## + +# %% EDSS Confusion Matrix +import pandas as pd +import numpy as np +import json +import glob +import os +import matplotlib.pyplot as plt +from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay + +def categorize_edss(value): + if pd.isna(value): + return np.nan + elif value <= 1.0: + return '0-1' + elif value <= 2.0: + return '1-2' + elif value <= 3.0: + return '2-3' + elif value <= 4.0: + return '3-4' + elif value <= 5.0: + return '4-5' + elif value <= 6.0: + return '5-6' + elif value <= 7.0: + return '6-7' + elif value <= 8.0: + return '7-8' + elif value <= 9.0: + return '8-9' + elif value <= 10.0: + return '9-10' + else: + return '10+' + +def plot_categorized_edss(json_dir_path, ground_truth_path): + # 1. Load Ground Truth + df_gt = pd.read_csv(ground_truth_path, sep=';') + df_gt['unique_id'] = df_gt['unique_id'].astype(str) + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str) + df_gt['EDSS'] = pd.to_numeric(df_gt['EDSS'], errors='coerce') + + # 2. Iterate through JSON files + all_preds = [] + json_pattern = os.path.join(json_dir_path, "*.json") + for file_path in glob.glob(json_pattern): + with open(file_path, 'r', encoding='utf-8') as f: + try: + data = json.load(f) + for entry in data: + if entry.get("success") and "result" in entry: + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')), + 'MedDatum': str(res.get('MedDatum')), + 'edss_pred': res.get('EDSS') + }) + except Exception as e: + print(f"Error reading {file_path}: {e}") + + df_pred = pd.DataFrame(all_preds) + df_pred['edss_pred'] = pd.to_numeric(df_pred['edss_pred'], errors='coerce') + + # 3. Merge and Categorize + # Clean keys to ensure 100% match rate + for df in [df_gt, df_pred]: + df['unique_id'] = df['unique_id'].astype(str).str.strip() + df['MedDatum'] = df['MedDatum'].astype(str).str.strip() + + df_merged = pd.merge( + df_gt[['unique_id', 'MedDatum', 'EDSS']], + df_pred, + on=['unique_id', 'MedDatum'], + how='inner' + ) + + df_merged = df_merged.dropna(subset=['EDSS', 'edss_pred']) + + # --- ADDED THESE LINES TO FIX THE NAMEERROR --- + y_true = df_merged['EDSS'].apply(categorize_edss) + y_pred = df_merged['edss_pred'].apply(categorize_edss) + # ---------------------------------------------- + + print(f"Verification: Total matches in Confusion Matrix: {len(df_merged)}") + + # 4. Define fixed labels to handle data gaps + fixed_labels = ['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'] + + # 5. Generate Confusion Matrix + cm = confusion_matrix(y_true, y_pred, labels=fixed_labels) + + # 6. Plotting + fig, ax = plt.subplots(figsize=(10, 8)) + disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=fixed_labels) + + # Plotting (y_axis is Ground Truth, x_axis is LLM Prediction) + disp.plot(cmap=plt.cm.Blues, values_format='d', ax=ax) + + plt.title('Categorized EDSS: Ground Truth vs LLM Prediction') + plt.ylabel('Ground Truth EDSS') + plt.xlabel('LLM Prediction') + plt.show() +## + +# %% Confusion Matrix adjusted +import pandas as pd +import numpy as np +import json +import glob +import os +import matplotlib.pyplot as plt +from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay + +def categorize_edss(value): + """Bins EDSS values into clinical categories.""" + if pd.isna(value): + return np.nan + elif value <= 1.0: return '0-1' + elif value <= 2.0: return '1-2' + elif value <= 3.0: return '2-3' + elif value <= 4.0: return '3-4' + elif value <= 5.0: return '4-5' + elif value <= 6.0: return '5-6' + elif value <= 7.0: return '6-7' + elif value <= 8.0: return '7-8' + elif value <= 9.0: return '8-9' + elif value <= 10.0: return '9-10' + else: return '10+' + +def plot_categorized_edss(json_dir_path, ground_truth_path): + # 1. Load Ground Truth with Normalization + df_gt = pd.read_csv(ground_truth_path, sep=';') + # Standardize keys to ensure 1:N matching works + df_gt['unique_id'] = df_gt['unique_id'].astype(str).str.strip().str.lower() + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str).str.strip().str.lower() + df_gt['EDSS'] = pd.to_numeric(df_gt['EDSS'], errors='coerce') + + # 2. Load All Predictions from JSONs + all_preds = [] + json_files = glob.glob(os.path.join(json_dir_path, "*.json")) + + for file_path in json_files: + with open(file_path, 'r', encoding='utf-8') as f: + try: + data = json.load(f) + for entry in data: + # We only take 'success': true entries + if entry.get("success") and "result" in entry: + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')).strip().lower(), + 'MedDatum': str(res.get('MedDatum')).strip().lower(), + 'edss_pred': res.get('EDSS') + }) + except Exception as e: + print(f"Error reading {file_path}: {e}") + + df_pred = pd.DataFrame(all_preds) + df_pred['edss_pred'] = pd.to_numeric(df_pred['edss_pred'], errors='coerce') + + # 3. Merge (This should give you ~3934 rows based on your audit) + df_merged = pd.merge( + df_gt[['unique_id', 'MedDatum', 'EDSS']], + df_pred, + on=['unique_id', 'MedDatum'], + how='inner' + ) + + # --- THE BIG REVEAL: Count the NaNs --- + nan_in_gt = df_merged['EDSS'].isna().sum() + nan_in_pred = df_merged['edss_pred'].isna().sum() + + print("-" * 40) + print(f"TOTAL MERGED ROWS: {len(df_merged)}") + print(f"Rows with missing Ground Truth EDSS: {nan_in_gt}") + print(f"Rows with missing Prediction EDSS: {nan_in_pred}") + print("-" * 40) + + # Now drop rows that have NO values in either side for the matrix + df_final = df_merged.dropna(subset=['EDSS', 'edss_pred']).copy() + print(f"FINAL ROWS FOR CONFUSION MATRIX: {len(df_final)}") + print("-" * 40) + + # 4. Categorize for the Matrix + y_true = df_final['EDSS'].apply(categorize_edss) + y_pred = df_final['edss_pred'].apply(categorize_edss) + + fixed_labels = ['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'] + + # 5. Generate and Print Raw Matrix + cm = confusion_matrix(y_true, y_pred, labels=fixed_labels) + + # Print the Raw Matrix to terminal + cm_df = pd.DataFrame(cm, index=[f"True_{l}" for l in fixed_labels], + columns=[f"Pred_{l}" for l in fixed_labels]) + print("\nRAW CONFUSION MATRIX (Rows=True, Cols=Pred):") + print(cm_df) + + # 6. Plotting + fig, ax = plt.subplots(figsize=(12, 10)) + disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=fixed_labels) + + # Values_format='d' ensures we see whole numbers, not scientific notation + disp.plot(cmap=plt.cm.Blues, values_format='d', ax=ax) + + plt.title(f'EDSS Confusion Matrix\n(n={len(df_final)} iterations across ~400 cases)', fontsize=14) + plt.ylabel('Ground Truth (Clinician)') + plt.xlabel('LLM Prediction') + plt.xticks(rotation=45) + plt.tight_layout() + plt.show() + +## +# %% Subcategories + +import pandas as pd +import numpy as np +import json +import glob +import os +import matplotlib.pyplot as plt + +def plot_subcategory_analysis(json_dir_path, ground_truth_path): + # 1. Column Mapping (JSON Key : CSV Column) + mapping = { + "VISUAL_OPTIC_FUNCTIONS": "Sehvermögen", + "BRAINSTEM_FUNCTIONS": "Hirnstamm", + "PYRAMIDAL_FUNCTIONS": "Pyramidalmotorik", + "CEREBELLAR_FUNCTIONS": "Cerebellum", + "SENSORY_FUNCTIONS": "Sensibiliät", + "BOWEL_AND_BLADDER_FUNCTIONS": "Blasen-_und_Mastdarmfunktion", + "CEREBRAL_FUNCTIONS": "Cerebrale_Funktion", + "AMBULATION": "Ambulation" + } + + # 2. Load Ground Truth + df_gt = pd.read_csv(ground_truth_path, sep=';') + df_gt['unique_id'] = df_gt['unique_id'].astype(str) + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str) + + # 3. Load Predictions including Subcategories + all_preds = [] + for file_path in glob.glob(os.path.join(json_dir_path, "*.json")): + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + for entry in data: + if entry.get("success"): + res = entry["result"] + row = { + 'unique_id': str(res.get('unique_id')), + 'MedDatum': str(res.get('MedDatum')) + } + # Add subcategory scores + for json_key in mapping.keys(): + row[json_key] = res.get('subcategories', {}).get(json_key) + all_preds.append(row) + + df_pred = pd.DataFrame(all_preds) + + # 4. Merge + df_merged = pd.merge(df_gt, df_pred, on=['unique_id', 'MedDatum'], suffixes=('_gt', '_llm')) + + # 5. Calculate Metrics + results = [] + for json_key, csv_col in mapping.items(): + # Ensure numeric + true_vals = pd.to_numeric(df_merged[csv_col], errors='coerce') + pred_vals = pd.to_numeric(df_merged[json_key], errors='coerce') + + # Drop NaNs for this specific subcategory + mask = true_vals.notna() & pred_vals.notna() + y_t = true_vals[mask] + y_p = pred_vals[mask] + + if len(y_t) > 0: + accuracy = (y_t == y_p).mean() * 100 + mae = np.abs(y_t - y_p).mean() # Mean Absolute Error (Deviation) + results.append({ + 'Subcategory': csv_col, + 'Accuracy': accuracy, + 'Deviation': mae + }) + + stats_df = pd.DataFrame(results).sort_values('Accuracy', ascending=False) + +# 6. Plotting + fig, ax1 = plt.subplots(figsize=(14, 7)) + + # Bar chart for Accuracy + bars = ax1.bar(stats_df['Subcategory'], stats_df['Accuracy'], + color='#3498db', alpha=0.8, label='Accuracy (%)') + ax1.set_ylabel('Accuracy (%)', color='#2980b9', fontsize=12, fontweight='bold') + ax1.set_ylim(0, 115) # Extra head room for labels + ax1.grid(axis='y', linestyle='--', alpha=0.7) + + # Rotate labels + plt.xticks(rotation=30, ha='right', fontsize=10) + + # Line chart for Deviation + ax2 = ax1.twinx() + ax2.plot(stats_df['Subcategory'], stats_df['Deviation'], + color='#e74c3c', marker='o', linewidth=2.5, markersize=8, + label='Mean Abs. Deviation (Score Points)') + ax2.set_ylabel('Mean Absolute Deviation', color='#c0392b', fontsize=12, fontweight='bold') + + # Adjust ax2 limit to avoid overlap with accuracy text + ax2.set_ylim(0, max(stats_df['Deviation']) * 1.5 if not stats_df['Deviation'].empty else 1) + +# plt.title('Subcategory Performance: Accuracy vs. Mean Deviation', fontsize=14, pad=20) + + # --- THE FIX: Better Legend Placement --- + # Combine legends from both axes and place them above the plot + lines1, labels1 = ax1.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax1.legend(lines1 + lines2, labels1 + labels2, + loc='upper center', bbox_to_anchor=(0.5, 1.12), + ncol=2, frameon=False, fontsize=11) + + # Add percentage labels on top of bars + for bar in bars: + height = bar.get_height() + ax1.annotate(f'{height:.1f}%', + xy=(bar.get_x() + bar.get_width() / 2, height), + xytext=(0, 5), textcoords="offset points", + ha='center', va='bottom', fontweight='bold', color='#2c3e50') + + plt.tight_layout() + plt.show() +## + +# %% Certainty +import pandas as pd +import numpy as np +import json +import glob +import os +import matplotlib.pyplot as plt + +def categorize_edss(value): + if pd.isna(value): return np.nan + elif value <= 1.0: return '0-1' + elif value <= 2.0: return '1-2' + elif value <= 3.0: return '2-3' + elif value <= 4.0: return '3-4' + elif value <= 5.0: return '4-5' + elif value <= 6.0: return '5-6' + elif value <= 7.0: return '6-7' + elif value <= 8.0: return '7-8' + elif value <= 9.0: return '8-9' + elif value <= 10.0: return '9-10' + else: return '10+' + +def plot_certainty_vs_accuracy_by_category(json_dir_path, ground_truth_path): + # 1. Data Loading & Merging + df_gt = pd.read_csv(ground_truth_path, sep=';') + df_gt['unique_id'] = df_gt['unique_id'].astype(str) + df_gt['MedDatum'] = df_gt['MedDatum'].astype(str) + df_gt['EDSS_true'] = pd.to_numeric(df_gt['EDSS'], errors='coerce') + + all_preds = [] + for file_path in glob.glob(os.path.join(json_dir_path, "*.json")): + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + for entry in data: + if entry.get("success"): + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')), + 'MedDatum': str(res.get('MedDatum')), + 'EDSS_pred': res.get('EDSS'), + 'certainty': res.get('certainty_percent') + }) + + df_pred = pd.DataFrame(all_preds) + df_pred['EDSS_pred'] = pd.to_numeric(df_pred['EDSS_pred'], errors='coerce') + + df = pd.merge(df_gt[['unique_id', 'MedDatum', 'EDSS_true']], + df_pred, on=['unique_id', 'MedDatum']).dropna() + + # 2. Process Metrics + df['gt_category'] = df['EDSS_true'].apply(categorize_edss) + df['is_correct'] = (df['EDSS_true'].round(1) == df['EDSS_pred'].round(1)) + + fixed_labels = ['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'] + + # Calculate Mean Certainty and Mean Accuracy per category + stats = df.groupby('gt_category').agg({ + 'is_correct': 'mean', + 'certainty': 'mean', + 'unique_id': 'count' + }).reindex(fixed_labels) + + stats['accuracy_percent'] = stats['is_correct'] * 100 + stats = stats.fillna(0) + + # 3. Plotting + x = np.arange(len(fixed_labels)) + width = 0.35 # Width of the bars + + fig, ax = plt.subplots(figsize=(14, 8)) + + # Plotting both bars side-by-side + rects1 = ax.bar(x - width/2, stats['accuracy_percent'], width, + label='Actual Accuracy (%)', color='#2ecc71', alpha=0.8) + rects2 = ax.bar(x + width/2, stats['certainty'], width, + label='LLM Avg. Certainty (%)', color='#e67e22', alpha=0.8) + + # Add text labels, titles and custom x-axis tick labels, etc. + ax.set_ylabel('Percentage (%)', fontsize=12) + ax.set_xlabel('Ground Truth EDSS Category', fontsize=12) +# ax.set_title('Comparison: LLM Confidence (Certainty) vs. Real Accuracy per EDSS Range', fontsize=15, pad=25) + ax.set_xticks(x) + ax.set_xticklabels(fixed_labels) + ax.set_ylim(0, 115) + ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.08), ncol=2, frameon=False) + ax.grid(axis='y', linestyle=':', alpha=0.5) + + # Helper function to label bar heights + def autolabel(rects): + for rect in rects: + height = rect.get_height() + if height > 0: + ax.annotate(f'{height:.0f}%', + xy=(rect.get_x() + rect.get_width() / 2, height), + xytext=(0, 3), textcoords="offset points", + ha='center', va='bottom', fontsize=9, fontweight='bold') + + autolabel(rects1) + autolabel(rects2) + + # Add sample size (n) at the bottom + for i, count in enumerate(stats['unique_id']): + ax.text(i, 2, f'n={int(count)}', ha='center', va='bottom', fontsize=10, color='white', fontweight='bold') + + plt.tight_layout() + plt.show() + +## + + + + +# %% Audit + + +import pandas as pd +import numpy as np +import json +import glob +import os + +def audit_matches(json_dir_path, ground_truth_path): + # 1. Load GT + df_gt = pd.read_csv(ground_truth_path, sep=';') + + # 2. Advanced Normalization + def clean_series(s): + return s.astype(str).str.strip().str.lower() + + df_gt['unique_id'] = clean_series(df_gt['unique_id']) + df_gt['MedDatum'] = clean_series(df_gt['MedDatum']) + + # 3. Load Predictions + all_preds = [] + for file_path in glob.glob(os.path.join(json_dir_path, "*.json")): + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + file_name = os.path.basename(file_path) + for entry in data: + if entry.get("success"): + res = entry["result"] + all_preds.append({ + 'unique_id': str(res.get('unique_id')).strip().lower(), + 'MedDatum': str(res.get('MedDatum')).strip().lower(), + 'file': file_name + }) + + df_pred = pd.DataFrame(all_preds) + + # 4. Find the "Ghost" entries (In JSON but not in GT) + # Create a 'key' column for easy comparison + df_gt['key'] = df_gt['unique_id'] + "_" + df_gt['MedDatum'] + df_pred['key'] = df_pred['unique_id'] + "_" + df_pred['MedDatum'] + + gt_keys = set(df_gt['key']) + df_pred['is_matched'] = df_pred['key'].isin(gt_keys) + + unmatched_summary = df_pred[df_pred['is_matched'] == False] + + print("--- AUDIT RESULTS ---") + print(f"Total rows in JSON: {len(df_pred)}") + print(f"Rows that matched GT: {df_pred['is_matched'].sum()}") + print(f"Rows that FAILED to match: {len(unmatched_summary)}") + + if not unmatched_summary.empty: + print("\nFirst 10 Unmatched Entries (check these against your CSV):") + print(unmatched_summary[['unique_id', 'MedDatum', 'file']].head(10)) + + # Breakdown by file - see if specific JSON files are broken + print("\nFailure count per JSON file:") + print(unmatched_summary['file'].value_counts()) + + +## + +# %% Usage + +# --- Usage --- +#plot_categorized_edss('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', +# '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') + +#plot_subcategory_analysis('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') +plot_certainty_vs_accuracy_by_category('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') +#audit_matches('/home/shahin/Lab/Doktorarbeit/Barcelona/Data/iteration', '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/GT_Numbers.csv') + +## diff --git a/Data/show_plots.py b/Data/show_plots.py index 724f46a..49184ac 100644 --- a/Data/show_plots.py +++ b/Data/show_plots.py @@ -754,7 +754,7 @@ print("\nFirst few rows:") print(df.head()) # Hardcode specific patient names -patient_names = ['6b56865d'] +patient_names = ['2bf8486d'] # Define the functional systems (columns to plot) - adjust based on actual column names functional_systems = ['EDSS', 'Visual', 'Sensory', 'Motor', 'Brainstem', 'Cerebellar', 'Autonomic', 'Bladder', 'Intellectual'] diff --git a/certainty.py b/certainty.py new file mode 100644 index 0000000..cfc11be --- /dev/null +++ b/certainty.py @@ -0,0 +1,600 @@ + +# %% API call1 +#import time +#import json +#import os +#from datetime import datetime +#import pandas as pd +#from openai import OpenAI +#from dotenv import load_dotenv +# +## Load environment variables +#load_dotenv() +# +## === CONFIGURATION === +#OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +#OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL") +#MODEL_NAME = "GPT-OSS-120B" +#HEALTH_URL = f"{OPENAI_BASE_URL}/health" # Placeholder - actual health check would need to be implemented +#CHAT_URL = f"{OPENAI_BASE_URL}/chat/completions" +# +## File paths +#INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique.csv" +#EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt" +##GRAMMAR_FILE = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/just_edss_schema.gbnf" +# +## Initialize OpenAI client +#client = OpenAI( +# api_key=OPENAI_API_KEY, +# base_url=OPENAI_BASE_URL +#) +# +## Read EDSS instructions from file +#with open(EDSS_INSTRUCTIONS_PATH, 'r') as f: +# EDSS_INSTRUCTIONS = f.read().strip() +## === RUN INFERENCE 2 === +#def run_inference(patient_text): +# prompt = f''' +# Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale) aus klinischen Berichten zu extrahieren. +#### Regeln für die Ausgabe: +#1. **Reason**: Erstelle eine prägnante Zusammenfassung (max. 400 Zeichen) der Befunde auf **DEUTSCH**, die zur Einstufung führen. +#2. **klassifizierbar**: +# - Setze dies auf **true**, wenn ein EDSS-Wert identifiziert, berechnet oder basierend auf den klinischen Hinweisen plausibel geschätzt werden kann. +# - Setze dies auf **false**, NUR wenn die Daten absolut unzureichend oder so widersprüchlich sind, dass keinerlei Einstufung möglich ist. +#3. **EDSS**: +# - Dieses Feld ist **VERPFLICHTEND**, wenn "klassifizierbar" auf true steht. +# - Es muss eine Zahl zwischen 0.0 und 10.0 sein. +# - Versuche stets, den EDSS-Wert so präzise wie möglich zu bestimmen, auch wenn die Datenlage dünn ist (nutze verfügbare Informationen zu Gehstrecke und Funktionssystemen). +# - Dieses Feld **DARF NICHT ERSCHEINEN**, wenn "klassifizierbar" auf false steht. +# +#### Einschränkungen: +#- Erfinde keine Fakten, aber nutze klinische Herleitungen aus dem Bericht, um den EDSS zu bestimmen. +#- Priorisiere die Vergabe eines EDSS-Wertes gegenüber der Markierung als nicht klassifizierbar. +#- Halte dich strikt an die JSON-Struktur. +# +#EDSS-Bewertungsrichtlinien: +#{EDSS_INSTRUCTIONS} +# +#Patientenbericht: +#{patient_text} +#''' +# start_time = time.time() +# +# try: +# # Make API call using OpenAI client +# response = client.chat.completions.create( +# messages=[ +# { +# "role": "system", +# "content": "You extract EDSS scores. You prioritize providing a score even if data is partial, by using clinical inference." +# }, +# { +# "role": "user", +# "content": prompt +# } +# ], +# model=MODEL_NAME, +# max_tokens=2048, +# temperature=0.0, +# response_format={"type": "json_object"} +# ) +# +# # Extract content from response +# content = response.choices[0].message.content +# +# # Parse the JSON response +# parsed = json.loads(content) +# +# inference_time = time.time() - start_time +# +# return { +# "success": True, +# "result": parsed, +# "inference_time_sec": inference_time +# } +# +# except Exception as e: +# print(f"Inference error: {e}") +# return { +# "success": False, +# "error": str(e), +# "inference_time_sec": -1 +# } +## === BUILD PATIENT TEXT === +#def build_patient_text(row): +# return ( +# str(row["T_Zusammenfassung"]) + "\n" + +# str(row["Diagnosen"]) + "\n" + +# str(row["T_KlinBef"]) + "\n" + +# str(row["T_Befunde"]) + "\n" +# ) +# +#if __name__ == "__main__": +# # Read CSV file ONLY inside main block +# df = pd.read_csv(INPUT_CSV, sep=';') +# results = [] +# +# # Process each row +# for idx, row in df.iterrows(): +# print(f"Processing row {idx + 1}/{len(df)}") +# try: +# patient_text = build_patient_text(row) +# result = run_inference(patient_text) +# +# # Add unique_id and MedDatum to result for tracking +# result["unique_id"] = row.get("unique_id", f"row_{idx}") +# result["MedDatum"] = row.get("MedDatum", None) +# +# results.append(result) +# print(json.dumps(result, indent=2)) +# except Exception as e: +# print(f"Error processing row {idx}: {e}") +# results.append({ +# "success": False, +# "error": str(e), +# "unique_id": row.get("unique_id", f"row_{idx}"), +# "MedDatum": row.get("MedDatum", None) +# }) +# +# # Save results to a JSON file +# output_json = INPUT_CSV.replace(".csv", "_results_Nisch.json") +# with open(output_json, 'w') as f: +# json.dump(results, f, indent=2) +# print(f"Results saved to {output_json}") +## + + + +# %% API call1 - Enhanced with certainty scoring +#import time +#import json +#import os +#from datetime import datetime +#import pandas as pd +#from openai import OpenAI +#from dotenv import load_dotenv +# +## Load environment variables +#load_dotenv() +# +## === CONFIGURATION === +#OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +#OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL") +#MODEL_NAME = "GPT-OSS-120B" +# +## File paths +#INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Test.csv" +#EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt" +# +## Initialize OpenAI client +#client = OpenAI( +# api_key=OPENAI_API_KEY, +# base_url=OPENAI_BASE_URL +#) +# +## Read EDSS instructions from file +#with open(EDSS_INSTRUCTIONS_PATH, 'r') as f: +# EDSS_INSTRUCTIONS = f.read().strip() +# +## === PROMPT WITH CERTAINTY REQUEST === +#def build_prompt(patient_text): +# return f'''Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale), alle Unterkategorien und die Bewertungssicherheit aus klinischen Berichten zu extrahieren. +# +#### Deine Aufgabe: +#1. Analysiere den Patientenbericht und extrahiere: +# - Den Gesamt-EDSS-Score (0.0–10.0) +# - Alle 8 EDSS-Unterkategorien (mit jeweils eigener Maximalpunktzahl) +#2. Schätze für jede Entscheidung die Sicherheit als Ganzzahl von 0–100 % ein. +# +#### Struktur der JSON-Ausgabe (VERPFLICHTEND): +#Gib NUR gültiges JSON zurück — kein Markdown, kein Text davor/dahinter. +# +#{{ +# "reason": "Kernaussage zur EDSS-Begründung (max. 400 Zeichen, auf Deutsch).", +# "klassifizierbar": true/false, +# "EDSS": null ODER Zahl zwischen 0.0 und 10.0 (nur wenn klassifizierbar=true)", +# "certainty_percent": 0 ODER Zahl zwischen 0 und 100 (Ganzzahl)", +# "subcategories": {{ +# "VISUAL_OPTIC_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "BRAINSTEM_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "PYRAMIDAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "CEREBELLAR_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "SENSORY_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "BOWEL_AND_BLADDER_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "CEREBRAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, +# "AMBULATION": null ODER Zahl zwischen 0.0 und 10.0 +# }} +#}} +# +#### Regeln: +#- **reason**: Kurze, prägnante Begründung (auf Deutsch, max. 400 Zeichen), warum du den EDSS-Wert und die Unterkategorien so bewertest. +#- **klassifizierbar**: +# - `true`, wenn EDSS und mindestens die wichtigsten Unterkategorien *eindeutig ableitbar* oder *plausibel inferierbar* sind. +# - `false`, **nur**, wenn keine relevanten Daten vorliegen, oder diese so widersprüchlich/inkonsistent sind, dass keine vernünftige Einschätzung möglich ist. +#- **EDSS**: +# - **VERPFLICHTEND**, wenn `klassifizierbar=true`. +# - Zahl zwischen 0.0 und 10.0 (z.B. 3.0, 5.5). Darf **nicht** erscheinen, wenn `klassifizierbar=false`. +#- **certainty_percent**: +# - **Immer present** — Ganzzahl (0–100), basierend auf: +# - Klarheit und Vollständigkeit der Berichtsangaben, +# - Stichhaltigkeit der Schlussfolgerung (inkl. Inferenz), +# - Konsistenz zwischen den Unterkategorien. +#- **subcategories**: +# - **Immer present** — **alle 8 Unterkategorien** müssen enthalten sein. +# - Jeder Wert ist entweder: +# - `null` (wenn keine ausreichende Information vorliegt), **oder** +# - eine Zahl ≤ jeweiliger Obergrenze (z.B. Ambulation ≤ 10.0). +# - Wenn die Unterkategorie plausibel inferiert werden kann (auch indirekt), gib einen sinnvollen Wert ab. +# - Beispiel: Wenn „Gang mit Krückstock auf ebenem Boden bis 200 m“ steht, setze `AMBULATION: 5.5`. +# +#### EDSS-Bewertungsrichtlinien: +#{EDSS_INSTRUCTIONS} +# +#Patientenbericht: +#{patient_text} +#''' +# +## === INFERENCE FUNCTION === +#def run_inference(patient_text): +# prompt = build_prompt(patient_text) +# +# start_time = time.time() +# +# try: +# response = client.chat.completions.create( +# messages=[ +# {"role": "system", "content": "Du gibst EXKLUSIV gültiges JSON zurück — keine weiteren Erklärungen."} +# ] + [ +# {"role": "user", "content": prompt} +# ], +# model=MODEL_NAME, +# max_tokens=2048, +# temperature=0.1, # Slightly higher for more natural certainty estimation (still low for reliability) +# response_format={"type": "json_object"} +# ) +# +# content = response.choices[0].message.content +# +# # Parse and validate JSON +# try: +# parsed = json.loads(content) +# except json.JSONDecodeError as e: +# print(f"⚠️ JSON parsing failed: {e}") +# print("Raw response:", content[:500]) +# raise ValueError("Model did not return valid JSON") +# +# # Enforce required keys +# if "certainty_percent" not in parsed: +# print("⚠️ Missing 'certainty_percent' in output! Force-adding fallback.") +# parsed["certainty_percent"] = 0 # fallback +# elif not isinstance(parsed["certainty_percent"], (int, float)): +# parsed["certainty_percent"] = int(parsed["certainty_percent"]) +# +# # Clamp certainty to [0, 100] +# pct = parsed["certainty_percent"] +# parsed["certainty_percent"] =max(0, min(100, int(pct))) +# +# # Enforce EDSS rules: if not classifiable → remove EDSS +# if not parsed.get("klassifizierbar", False): +# if "EDSS" in parsed: +# del parsed["EDSS"] # per spec, must not appear if not classifiable +# else: +# if "EDSS" not in parsed: +# print("⚠️ 'klassifizierbar' is true but EDSS missing — adding fallback.") +# parsed["EDSS"] = 7.0 # last-resort fallback +# +# inference_time = time.time() - start_time +# +# return { +# "success": True, +# "result": parsed, +# "inference_time_sec": inference_time +# } +# +# except Exception as e: +# print(f"❌ Inference error: {e}") +# return { +# "success": False, +# "error": str(e), +# "inference_time_sec": -1, +# "result": None # no structured output +# } +# +## === BUILD PATIENT TEXT === +#def build_patient_text(row): +# return ( +# str(row.get("T_Zusammenfassung", "")) + "\n" + +# str(row.get("Diagnosen", "")) + "\n" + +# str(row.get("T_KlinBef", "")) + "\n" + +# str(row.get("T_Befunde", "")) +# ) +# +#if __name__ == "__main__": +# # Load data +# df = pd.read_csv(INPUT_CSV, sep=';') +# results = [] +# +# # Optional: limit for testing +# # df = df.head(3) +# +# print(f"Processing {len(df)} rows...") +# for idx, row in df.iterrows(): +# print(f"\n— Row {idx + 1}/{len(df)} —") +# try: +# patient_text = build_patient_text(row) +# result = run_inference(patient_text) +# +# # Attach metadata +# result["unique_id"] = row.get("unique_id", f"row_{idx}") +# result["MedDatum"] = row.get("MedDatum", None) +# +# results.append(result) +# +# # Print summary +# if result["success"]: +# res = result["result"] +# edss = res.get("EDSS", "N/A") if res.get("klassifizierbar") else "N/A" +# print(f"✅ Result → EDSS={edss}, certainty={res.get('certainty_percent', 'N/A')}%") +# print(f" Reason: {res.get('reason', 'N/A')[:100]}…") +# else: +# print(f"❌ Failed: {result.get('error', 'Unknown error')[:100]}") +# +# except Exception as e: +# print(f"⚠️ Error processing row {idx}: {e}") +# results.append({ +# "success": False, +# "error": str(e), +# "unique_id": row.get("unique_id", f"row_{idx}"), +# "MedDatum": row.get("MedDatum", None), +# "result": None +# }) +# +# # Save results +# output_json = INPUT_CSV.replace(".csv", "_results_Nisch_certainty.json") +# with open(output_json, 'w', encoding='utf-8') as f: +# json.dump(results, f, indent=2, ensure_ascii=False) +# print(f"\n✅ Saved results to: {output_json}") +# +## + + +# %% API call - Multi-iteration EDSS + certainty extraction + +import time +import json +import os +from datetime import datetime +import pandas as pd +from openai import OpenAI +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +# === CONFIGURATION === +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") +OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL") +MODEL_NAME = "GPT-OSS-120B" + +# File paths +INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique.csv" +EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt" + +# Iteration settings +NUM_ITERATIONS = 20 +STOP_ON_FIRST_ERROR = False # Set to True for debugging + +# Initialize OpenAI client +client = OpenAI( + api_key=OPENAI_API_KEY, + base_url=OPENAI_BASE_URL +) + +# Read EDSS instructions from file +with open(EDSS_INSTRUCTIONS_PATH, 'r') as f: + EDSS_INSTRUCTIONS = f.read().strip() + +# === PROMPT (unchanged from before) === +def build_prompt(patient_text): + return f'''Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale), alle Unterkategorien und die Bewertungssicherheit aus klinischen Berichten zu extrahieren. + +### Deine Aufgabe: +1. Analysiere den Patientenbericht und extrahiere: + - Den Gesamt-EDSS-Score (0.0–10.0) + - Alle 8 EDSS-Unterkategorien (mit jeweils eigener Maximalpunktzahl) +2. Schätze für jede Entscheidung die Sicherheit als Ganzzahl von 0–100 % ein. + +### Struktur der JSON-Ausgabe (VERPFLICHTEND): +Gib NUR gültiges JSON zurück — kein Markdown, kein Text davor/dahinter. + +{{ + "reason": "Kernaussage zur EDSS-Begründung (max. 400 Zeichen, auf Deutsch).", + "klassifizierbar": true/false, + "EDSS": null ODER Zahl zwischen 0.0 und 10.0 (nur wenn klassifizierbar=true)", + "certainty_percent": 0 ODER Zahl zwischen 0 und 100 (Ganzzahl)", + "subcategories": {{ + "VISUAL_OPTIC_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "BRAINSTEM_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "PYRAMIDAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "CEREBELLAR_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "SENSORY_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "BOWEL_AND_BLADDER_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "CEREBRAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0, + "AMBULATION": null ODER Zahl zwischen 0.0 und 10.0 + }} +}} + +### Regeln: +- **reason**: Kurze, prägnante Begründung (auf Deutsch, max. 400 Zeichen), warum du den EDSS-Wert und die Unterkategorien so bewertest. +- **klassifizierbar**: + - `true`, wenn EDSS und mindestens die wichtigsten Unterkategorien *eindeutig ableitbar* oder *plausibel inferierbar* sind. + - `false`, **nur**, wenn keine relevanten Daten vorliegen, oder diese so widersprüchlich/inkonsistent sind, dass keine vernünftige Einschätzung möglich ist. +- **EDSS**: + - **VERPFLICHTEND**, wenn `klassifizierbar=true`. + - Zahl zwischen 0.0 und 10.0 (z.B. 3.0, 5.5). Darf **nicht** erscheinen, wenn `klassifizierbar=false`. +- **certainty_percent**: + - **Immer present** — Ganzzahl (0–100), basierend auf: + - Klarheit und Vollständigkeit der Berichtsangaben, + - Stichhaltigkeit der Schlussfolgerung (inkl. Inferenz), + - Konsistenz zwischen den Unterkategorien. +- **subcategories**: + - **Immer present** — **alle 8 Unterkategorien** müssen enthalten sein. + - Jeder Wert ist entweder: + - `null` (wenn keine ausreichende Information vorliegt), **oder** + - eine Zahl ≤ jeweiliger Obergrenze (z.B. Ambulation ≤ 10.0). + - Wenn die Unterkategorie plausibel inferiert werden kann (auch indirekt), gib einen sinnvollen Wert ab. + - Beispiel: Wenn „Gang mit Krückstock auf ebenem Boden bis 200 m“ steht, setze `AMBULATION: 5.5`. + +### EDSS-Bewertungsrichtlinien: +{EDSS_INSTRUCTIONS} + +Patientenbericht: +{patient_text} +''' + +# === INFERENCE FUNCTION (unchanged) === +def run_inference(patient_text): + prompt = build_prompt(patient_text) + + start_time = time.time() + + try: + response = client.chat.completions.create( + messages=[ + {"role": "system", "content": "Du gibst EXKLUSIV gültiges JSON zurück — keine weiteren Erklärungen."} + ] + [ + {"role": "user", "content": prompt} + ], + model=MODEL_NAME, + max_tokens=2048, + temperature=0.1, + response_format={"type": "json_object"} + ) + + content = response.choices[0].message.content + + # Parse and validate JSON + try: + parsed = json.loads(content) + except json.JSONDecodeError as e: + print(f"⚠️ JSON parsing failed: {e}") + print("Raw response:", content[:500]) + raise ValueError("Model did not return valid JSON") + + # Enforce required keys + if "certainty_percent" not in parsed: + print("⚠️ Missing 'certainty_percent' in output! Force-adding fallback.") + parsed["certainty_percent"] = 0 + elif not isinstance(parsed["certainty_percent"], (int, float)): + parsed["certainty_percent"] = int(parsed["certainty_percent"]) + + # Clamp certainty to [0, 100] + pct = parsed["certainty_percent"] + parsed["certainty_percent"] = max(0, min(100, int(pct))) + + # Enforce EDSS rules + if not parsed.get("klassifizierbar", False): + if "EDSS" in parsed: + del parsed["EDSS"] + else: + if "EDSS" not in parsed: + print("⚠️ 'klassifizierbar' is true but EDSS missing — adding fallback.") + parsed["EDSS"] = 7.0 + + inference_time = time.time() - start_time + + return { + "success": True, + "result": parsed, + "inference_time_sec": inference_time + } + + except Exception as e: + print(f"❌ Inference error: {e}") + return { + "success": False, + "error": str(e), + "inference_time_sec": -1, + "result": None + } + +# === BUILD PATIENT TEXT === +def build_patient_text(row): + return ( + str(row.get("T_Zusammenfassung", "")) + "\n" + + str(row.get("Diagnosen", "")) + "\n" + + str(row.get("T_KlinBef", "")) + "\n" + + str(row.get("T_Befunde", "")) + ) + +# === MAIN LOOP (NEW: MULTI-ITERATION) === +if __name__ == "__main__": + # Load data ONCE (to avoid repeated I/O overhead) + df = pd.read_csv(INPUT_CSV, sep=';') + total_rows = len(df) + print(f"Loaded {total_rows} patient records.") + + for iteration in range(1, NUM_ITERATIONS + 1): + print(f"\n{'='*60}") + print(f"🔄 ITERATION {iteration}/{NUM_ITERATIONS}") + print(f"{'='*60}") + + iteration_results = [] + start_iter = time.time() + + for idx, row in df.iterrows(): + print(f"\rRow {idx+1}/{total_rows} | Iter {iteration}", end='', flush=True) + try: + patient_text = build_patient_text(row) + result = run_inference(patient_text) + + # Attach metadata + if result["success"]: + res = result["result"].copy() # avoid mutation + res["iteration"] = iteration + res["unique_id"] = row.get("unique_id", f"row_{idx}") + res["MedDatum"] = row.get("MedDatum", None) + result["result"] = res + + else: + result["iteration"] = iteration + result["unique_id"] = row.get("unique_id", f"row_{idx}") + result["MedDatum"] = row.get("MedDatum", None) + + iteration_results.append(result) + + if result["success"]: + res = result["result"] + edss = res.get("EDSS", "N/A") if res.get("klassifizierbar") else "N/A" + print(f" ✅ EDSS={edss}, cert={res.get('certainty_percent', '?')}%") + else: + print(f" ❌ {result.get('error', 'Unknown')}") + + except Exception as e: + print(f"\n⚠️ Row {idx} failed: {e}") + iteration_results.append({ + "success": False, + "error": str(e), + "iteration": iteration, + "unique_id": row.get("unique_id", f"row_{idx}"), + "MedDatum": row.get("MedDatum", None), + "result": None + }) + if STOP_ON_FIRST_ERROR: + break + + # Save per-iteration results + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_path = INPUT_CSV.replace(".csv", f"_results_iter_{iteration}_{timestamp}.json") + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(iteration_results, f, indent=2, ensure_ascii=False) + print(f"\n✅ Iteration {iteration} complete. Saved to: {output_path}") + + elapsed = time.time() - start_iter + print(f"⏱️ Iteration {iteration} took {elapsed:.1f}s ({elapsed/total_rows:.1f}s/row)") + + print(f"\n🎉 All {NUM_ITERATIONS} iterations completed!") + + + +##