Update the Main Branch from certainty

.gitignore

@@ -1,12 +1,21 @@
+<<<<<<< HEAD
+# Ignore all contents of these directories
+!**/*.py
+/Data/
+/attach/
+/results/
+/enarcelona/
+=======
 # 1. Broad Ignores
 /Data/*
 /attach/*
 /results/*
 /enarcelona/*
+>>>>>>> Certainty
 .env
 __pycache__/
 *.pyc
-
+*.csv
 =======
 /reference/
 *.svg

.gitignore.orig

@@ -0,0 +1,29 @@
+<<<<<<< HEAD
+# Ignore all contents of these directories
+!**/*.py
+/Data/
+/attach/
+/results/
+/enarcelona/
+=======
+# 1. Broad Ignores
+/Data/*
+/attach/*
+/results/*
+/enarcelona/*
+>>>>>>> Certainty
+.env
+__pycache__/
+*.pyc
+*.csv
+=======
+/reference/
+*.svg
+>>>>>>> Stashed changes
+# 2. Ignore virtual environments COMPLETELY
+# This must come BEFORE the unignore rule
+env*/
+
+# 3. The "Unignore" rule (Whitelisting)
+# We only unignore .py files that aren't already blocked by the rules above
+!**/*.py

certainty.py

@@ -0,0 +1,600 @@
+
+# %% API call1
+#import time
+#import json
+#import os
+#from datetime import datetime
+#import pandas as pd
+#from openai import OpenAI
+#from dotenv import load_dotenv
+#
+## Load environment variables
+#load_dotenv()
+#
+## === CONFIGURATION ===
+#OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+#OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL")
+#MODEL_NAME = "GPT-OSS-120B"
+#HEALTH_URL = f"{OPENAI_BASE_URL}/health"  # Placeholder - actual health check would need to be implemented
+#CHAT_URL = f"{OPENAI_BASE_URL}/chat/completions"
+#
+## File paths
+#INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique.csv"
+#EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt"
+##GRAMMAR_FILE = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/just_edss_schema.gbnf"
+#
+## Initialize OpenAI client
+#client = OpenAI(
+#    api_key=OPENAI_API_KEY,
+#    base_url=OPENAI_BASE_URL
+#)
+#
+## Read EDSS instructions from file
+#with open(EDSS_INSTRUCTIONS_PATH, 'r') as f:
+#    EDSS_INSTRUCTIONS = f.read().strip()
+## === RUN INFERENCE 2 ===
+#def run_inference(patient_text):
+#    prompt = f'''
+#    Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale) aus klinischen Berichten zu extrahieren.
+#### Regeln für die Ausgabe:
+#1. **Reason**: Erstelle eine prägnante Zusammenfassung (max. 400 Zeichen) der Befunde auf **DEUTSCH**, die zur Einstufung führen.
+#2. **klassifizierbar**:
+#   - Setze dies auf **true**, wenn ein EDSS-Wert identifiziert, berechnet oder basierend auf den klinischen Hinweisen plausibel geschätzt werden kann.
+#   - Setze dies auf **false**, NUR wenn die Daten absolut unzureichend oder so widersprüchlich sind, dass keinerlei Einstufung möglich ist.
+#3. **EDSS**:
+#   - Dieses Feld ist **VERPFLICHTEND**, wenn "klassifizierbar" auf true steht.
+#   - Es muss eine Zahl zwischen 0.0 und 10.0 sein.
+#   - Versuche stets, den EDSS-Wert so präzise wie möglich zu bestimmen, auch wenn die Datenlage dünn ist (nutze verfügbare Informationen zu Gehstrecke und Funktionssystemen).
+#   - Dieses Feld **DARF NICHT ERSCHEINEN**, wenn "klassifizierbar" auf false steht.
+#
+#### Einschränkungen:
+#- Erfinde keine Fakten, aber nutze klinische Herleitungen aus dem Bericht, um den EDSS zu bestimmen.
+#- Priorisiere die Vergabe eines EDSS-Wertes gegenüber der Markierung als nicht klassifizierbar.
+#- Halte dich strikt an die JSON-Struktur.
+#
+#EDSS-Bewertungsrichtlinien:
+#{EDSS_INSTRUCTIONS}
+#
+#Patientenbericht:
+#{patient_text}
+#'''
+#    start_time = time.time()
+#
+#    try:
+#        # Make API call using OpenAI client
+#        response = client.chat.completions.create(
+#            messages=[
+#                {
+#                    "role": "system",
+#                    "content": "You extract EDSS scores. You prioritize providing a score even if data is partial, by using clinical inference."
+#                },
+#                {
+#                    "role": "user",
+#                    "content": prompt
+#                }
+#            ],
+#            model=MODEL_NAME,
+#            max_tokens=2048,
+#            temperature=0.0,
+#            response_format={"type": "json_object"}
+#        )
+#
+#        # Extract content from response
+#        content = response.choices[0].message.content
+#
+#        # Parse the JSON response
+#        parsed = json.loads(content)
+#
+#        inference_time = time.time() - start_time
+#
+#        return {
+#            "success": True,
+#            "result": parsed,
+#            "inference_time_sec": inference_time
+#        }
+#
+#    except Exception as e:
+#        print(f"Inference error: {e}")
+#        return {
+#            "success": False,
+#            "error": str(e),
+#            "inference_time_sec": -1
+#        }
+## === BUILD PATIENT TEXT ===
+#def build_patient_text(row):
+#    return (
+#        str(row["T_Zusammenfassung"]) + "\n" +
+#        str(row["Diagnosen"]) + "\n" +
+#        str(row["T_KlinBef"]) + "\n" +
+#        str(row["T_Befunde"]) + "\n"
+#    )
+#
+#if __name__ == "__main__":
+#    # Read CSV file ONLY inside main block
+#    df = pd.read_csv(INPUT_CSV, sep=';')
+#    results = []
+#
+#    # Process each row
+#    for idx, row in df.iterrows():
+#        print(f"Processing row {idx + 1}/{len(df)}")
+#        try:
+#            patient_text = build_patient_text(row)
+#            result = run_inference(patient_text)
+#
+#            # Add unique_id and MedDatum to result for tracking
+#            result["unique_id"] = row.get("unique_id", f"row_{idx}")
+#            result["MedDatum"] = row.get("MedDatum", None)
+#
+#            results.append(result)
+#            print(json.dumps(result, indent=2))
+#        except Exception as e:
+#            print(f"Error processing row {idx}: {e}")
+#            results.append({
+#                "success": False,
+#                "error": str(e),
+#                "unique_id": row.get("unique_id", f"row_{idx}"),
+#                "MedDatum": row.get("MedDatum", None)
+#            })
+#
+#    # Save results to a JSON file
+#    output_json = INPUT_CSV.replace(".csv", "_results_Nisch.json")
+#    with open(output_json, 'w') as f:
+#        json.dump(results, f, indent=2)
+#    print(f"Results saved to {output_json}")
+##
+
+
+
+# %% API call1 - Enhanced with certainty scoring
+#import time
+#import json
+#import os
+#from datetime import datetime
+#import pandas as pd
+#from openai import OpenAI
+#from dotenv import load_dotenv
+#
+## Load environment variables
+#load_dotenv()
+#
+## === CONFIGURATION ===
+#OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+#OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL")
+#MODEL_NAME = "GPT-OSS-120B"
+#
+## File paths
+#INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Test.csv"
+#EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt"
+#
+## Initialize OpenAI client
+#client = OpenAI(
+#    api_key=OPENAI_API_KEY,
+#    base_url=OPENAI_BASE_URL
+#)
+#
+## Read EDSS instructions from file
+#with open(EDSS_INSTRUCTIONS_PATH, 'r') as f:
+#    EDSS_INSTRUCTIONS = f.read().strip()
+#
+## === PROMPT WITH CERTAINTY REQUEST ===
+#def build_prompt(patient_text):
+#    return f'''Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale), alle Unterkategorien und die Bewertungssicherheit aus klinischen Berichten zu extrahieren.
+#
+#### Deine Aufgabe:
+#1. Analysiere den Patientenbericht und extrahiere:
+#   - Den Gesamt-EDSS-Score (0.0–10.0)
+#   - Alle 8 EDSS-Unterkategorien (mit jeweils eigener Maximalpunktzahl)
+#2. Schätze für jede Entscheidung die Sicherheit als Ganzzahl von 0–100 % ein.
+#
+#### Struktur der JSON-Ausgabe (VERPFLICHTEND):
+#Gib NUR gültiges JSON zurück — kein Markdown, kein Text davor/dahinter.
+#
+#{{
+#  "reason": "Kernaussage zur EDSS-Begründung (max. 400 Zeichen, auf Deutsch).",
+#  "klassifizierbar": true/false,
+#  "EDSS": null ODER Zahl zwischen 0.0 und 10.0 (nur wenn klassifizierbar=true)",
+#  "certainty_percent": 0 ODER Zahl zwischen 0 und 100 (Ganzzahl)",
+#  "subcategories": {{
+#    "VISUAL_OPTIC_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "BRAINSTEM_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "PYRAMIDAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "CEREBELLAR_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "SENSORY_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "BOWEL_AND_BLADDER_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "CEREBRAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+#    "AMBULATION": null ODER Zahl zwischen 0.0 und 10.0
+#  }}
+#}}
+#
+#### Regeln:
+#- **reason**: Kurze, prägnante Begründung (auf Deutsch, max. 400 Zeichen), warum du den EDSS-Wert und die Unterkategorien so bewertest.
+#- **klassifizierbar**:
+#  - `true`, wenn EDSS und mindestens die wichtigsten Unterkategorien *eindeutig ableitbar* oder *plausibel inferierbar* sind.
+#  - `false`, **nur**, wenn keine relevanten Daten vorliegen, oder diese so widersprüchlich/inkonsistent sind, dass keine vernünftige Einschätzung möglich ist.
+#- **EDSS**:
+#  - **VERPFLICHTEND**, wenn `klassifizierbar=true`.
+#  - Zahl zwischen 0.0 und 10.0 (z.B. 3.0, 5.5). Darf **nicht** erscheinen, wenn `klassifizierbar=false`.
+#- **certainty_percent**:
+#  - **Immer present** — Ganzzahl (0–100), basierend auf:
+#    - Klarheit und Vollständigkeit der Berichtsangaben,
+#    - Stichhaltigkeit der Schlussfolgerung (inkl. Inferenz),
+#    - Konsistenz zwischen den Unterkategorien.
+#- **subcategories**:
+#  - **Immer present** — **alle 8 Unterkategorien** müssen enthalten sein.
+#  - Jeder Wert ist entweder:
+#    - `null` (wenn keine ausreichende Information vorliegt), **oder**
+#    - eine Zahl ≤ jeweiliger Obergrenze (z.B. Ambulation ≤ 10.0).
+#  - Wenn die Unterkategorie plausibel inferiert werden kann (auch indirekt), gib einen sinnvollen Wert ab.
+#  - Beispiel: Wenn „Gang mit Krückstock auf ebenem Boden bis 200 m“ steht, setze `AMBULATION: 5.5`.
+#
+#### EDSS-Bewertungsrichtlinien:
+#{EDSS_INSTRUCTIONS}
+#
+#Patientenbericht:
+#{patient_text}
+#'''
+#
+## === INFERENCE FUNCTION ===
+#def run_inference(patient_text):
+#    prompt = build_prompt(patient_text)
+#
+#    start_time = time.time()
+#
+#    try:
+#        response = client.chat.completions.create(
+#            messages=[
+#                {"role": "system", "content": "Du gibst EXKLUSIV gültiges JSON zurück — keine weiteren Erklärungen."}
+#            ] + [
+#                {"role": "user", "content": prompt}
+#            ],
+#            model=MODEL_NAME,
+#            max_tokens=2048,
+#            temperature=0.1,  # Slightly higher for more natural certainty estimation (still low for reliability)
+#            response_format={"type": "json_object"}
+#        )
+#
+#        content = response.choices[0].message.content
+#
+#        # Parse and validate JSON
+#        try:
+#            parsed = json.loads(content)
+#        except json.JSONDecodeError as e:
+#            print(f"⚠️  JSON parsing failed: {e}")
+#            print("Raw response:", content[:500])
+#            raise ValueError("Model did not return valid JSON")
+#
+#        # Enforce required keys
+#        if "certainty_percent" not in parsed:
+#            print("⚠️  Missing 'certainty_percent' in output! Force-adding fallback.")
+#            parsed["certainty_percent"] = 0  # fallback
+#        elif not isinstance(parsed["certainty_percent"], (int, float)):
+#            parsed["certainty_percent"] = int(parsed["certainty_percent"])
+#
+#        # Clamp certainty to [0, 100]
+#        pct = parsed["certainty_percent"]
+#        parsed["certainty_percent"] =max(0, min(100, int(pct)))
+#
+#        # Enforce EDSS rules: if not classifiable → remove EDSS
+#        if not parsed.get("klassifizierbar", False):
+#            if "EDSS" in parsed:
+#                del parsed["EDSS"]  # per spec, must not appear if not classifiable
+#        else:
+#            if "EDSS" not in parsed:
+#                print("⚠️  'klassifizierbar' is true but EDSS missing — adding fallback.")
+#                parsed["EDSS"] = 7.0  # last-resort fallback
+#
+#        inference_time = time.time() - start_time
+#
+#        return {
+#            "success": True,
+#            "result": parsed,
+#            "inference_time_sec": inference_time
+#        }
+#
+#    except Exception as e:
+#        print(f"❌ Inference error: {e}")
+#        return {
+#            "success": False,
+#            "error": str(e),
+#            "inference_time_sec": -1,
+#            "result": None  # no structured output
+#        }
+#
+## === BUILD PATIENT TEXT ===
+#def build_patient_text(row):
+#    return (
+#        str(row.get("T_Zusammenfassung", "")) + "\n" +
+#        str(row.get("Diagnosen", "")) + "\n" +
+#        str(row.get("T_KlinBef", "")) + "\n" +
+#        str(row.get("T_Befunde", ""))
+#    )
+#
+#if __name__ == "__main__":
+#    # Load data
+#    df = pd.read_csv(INPUT_CSV, sep=';')
+#    results = []
+#
+#    # Optional: limit for testing
+#    # df = df.head(3)
+#
+#    print(f"Processing {len(df)} rows...")
+#    for idx, row in df.iterrows():
+#        print(f"\n— Row {idx + 1}/{len(df)} —")
+#        try:
+#            patient_text = build_patient_text(row)
+#            result = run_inference(patient_text)
+#
+#            # Attach metadata
+#            result["unique_id"] = row.get("unique_id", f"row_{idx}")
+#            result["MedDatum"] = row.get("MedDatum", None)
+#
+#            results.append(result)
+#
+#            # Print summary
+#            if result["success"]:
+#                res = result["result"]
+#                edss = res.get("EDSS", "N/A") if res.get("klassifizierbar") else "N/A"
+#                print(f"✅ Result → EDSS={edss}, certainty={res.get('certainty_percent', 'N/A')}%")
+#                print(f"   Reason: {res.get('reason', 'N/A')[:100]}…")
+#            else:
+#                print(f"❌ Failed: {result.get('error', 'Unknown error')[:100]}")
+#
+#        except Exception as e:
+#            print(f"⚠️  Error processing row {idx}: {e}")
+#            results.append({
+#                "success": False,
+#                "error": str(e),
+#                "unique_id": row.get("unique_id", f"row_{idx}"),
+#                "MedDatum": row.get("MedDatum", None),
+#                "result": None
+#            })
+#
+#    # Save results
+#    output_json = INPUT_CSV.replace(".csv", "_results_Nisch_certainty.json")
+#    with open(output_json, 'w', encoding='utf-8') as f:
+#        json.dump(results, f, indent=2, ensure_ascii=False)
+#    print(f"\n✅ Saved results to: {output_json}")
+#
+##
+
+
+# %% API call - Multi-iteration EDSS + certainty extraction
+
+import time
+import json
+import os
+from datetime import datetime
+import pandas as pd
+from openai import OpenAI
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# === CONFIGURATION ===
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL")
+MODEL_NAME = "GPT-OSS-120B"
+
+# File paths
+INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique.csv"
+EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt"
+
+# Iteration settings
+NUM_ITERATIONS = 20
+STOP_ON_FIRST_ERROR = False  # Set to True for debugging
+
+# Initialize OpenAI client
+client = OpenAI(
+    api_key=OPENAI_API_KEY,
+    base_url=OPENAI_BASE_URL
+)
+
+# Read EDSS instructions from file
+with open(EDSS_INSTRUCTIONS_PATH, 'r') as f:
+    EDSS_INSTRUCTIONS = f.read().strip()
+
+# === PROMPT (unchanged from before) ===
+def build_prompt(patient_text):
+    return f'''Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale), alle Unterkategorien und die Bewertungssicherheit aus klinischen Berichten zu extrahieren.
+
+### Deine Aufgabe:
+1. Analysiere den Patientenbericht und extrahiere:
+   - Den Gesamt-EDSS-Score (0.0–10.0)
+   - Alle 8 EDSS-Unterkategorien (mit jeweils eigener Maximalpunktzahl)
+2. Schätze für jede Entscheidung die Sicherheit als Ganzzahl von 0–100 % ein.
+
+### Struktur der JSON-Ausgabe (VERPFLICHTEND):
+Gib NUR gültiges JSON zurück — kein Markdown, kein Text davor/dahinter.
+
+{{
+  "reason": "Kernaussage zur EDSS-Begründung (max. 400 Zeichen, auf Deutsch).",
+  "klassifizierbar": true/false,
+  "EDSS": null ODER Zahl zwischen 0.0 und 10.0 (nur wenn klassifizierbar=true)",
+  "certainty_percent": 0 ODER Zahl zwischen 0 und 100 (Ganzzahl)",
+  "subcategories": {{
+    "VISUAL_OPTIC_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "BRAINSTEM_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "PYRAMIDAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "CEREBELLAR_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "SENSORY_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "BOWEL_AND_BLADDER_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "CEREBRAL_FUNCTIONS": null ODER Zahl zwischen 0.0 und 6.0,
+    "AMBULATION": null ODER Zahl zwischen 0.0 und 10.0
+  }}
+}}
+
+### Regeln:
+- **reason**: Kurze, prägnante Begründung (auf Deutsch, max. 400 Zeichen), warum du den EDSS-Wert und die Unterkategorien so bewertest.
+- **klassifizierbar**:
+  - `true`, wenn EDSS und mindestens die wichtigsten Unterkategorien *eindeutig ableitbar* oder *plausibel inferierbar* sind.
+  - `false`, **nur**, wenn keine relevanten Daten vorliegen, oder diese so widersprüchlich/inkonsistent sind, dass keine vernünftige Einschätzung möglich ist.
+- **EDSS**:
+  - **VERPFLICHTEND**, wenn `klassifizierbar=true`.
+  - Zahl zwischen 0.0 und 10.0 (z.B. 3.0, 5.5). Darf **nicht** erscheinen, wenn `klassifizierbar=false`.
+- **certainty_percent**:
+  - **Immer present** — Ganzzahl (0–100), basierend auf:
+    - Klarheit und Vollständigkeit der Berichtsangaben,
+    - Stichhaltigkeit der Schlussfolgerung (inkl. Inferenz),
+    - Konsistenz zwischen den Unterkategorien.
+- **subcategories**:
+  - **Immer present** — **alle 8 Unterkategorien** müssen enthalten sein.
+  - Jeder Wert ist entweder:
+    - `null` (wenn keine ausreichende Information vorliegt), **oder**
+    - eine Zahl ≤ jeweiliger Obergrenze (z.B. Ambulation ≤ 10.0).
+  - Wenn die Unterkategorie plausibel inferiert werden kann (auch indirekt), gib einen sinnvollen Wert ab.
+  - Beispiel: Wenn „Gang mit Krückstock auf ebenem Boden bis 200 m“ steht, setze `AMBULATION: 5.5`.
+
+### EDSS-Bewertungsrichtlinien:
+{EDSS_INSTRUCTIONS}
+
+Patientenbericht:
+{patient_text}
+'''
+
+# === INFERENCE FUNCTION (unchanged) ===
+def run_inference(patient_text):
+    prompt = build_prompt(patient_text)
+
+    start_time = time.time()
+
+    try:
+        response = client.chat.completions.create(
+            messages=[
+                {"role": "system", "content": "Du gibst EXKLUSIV gültiges JSON zurück — keine weiteren Erklärungen."}
+            ] + [
+                {"role": "user", "content": prompt}
+            ],
+            model=MODEL_NAME,
+            max_tokens=2048,
+            temperature=0.1,
+            response_format={"type": "json_object"}
+        )
+
+        content = response.choices[0].message.content
+
+        # Parse and validate JSON
+        try:
+            parsed = json.loads(content)
+        except json.JSONDecodeError as e:
+            print(f"⚠️   JSON parsing failed: {e}")
+            print("Raw response:", content[:500])
+            raise ValueError("Model did not return valid JSON")
+
+        # Enforce required keys
+        if "certainty_percent" not in parsed:
+            print("⚠️   Missing 'certainty_percent' in output! Force-adding fallback.")
+            parsed["certainty_percent"] = 0
+        elif not isinstance(parsed["certainty_percent"], (int, float)):
+            parsed["certainty_percent"] = int(parsed["certainty_percent"])
+
+        # Clamp certainty to [0, 100]
+        pct = parsed["certainty_percent"]
+        parsed["certainty_percent"] = max(0, min(100, int(pct)))
+
+        # Enforce EDSS rules
+        if not parsed.get("klassifizierbar", False):
+            if "EDSS" in parsed:
+                del parsed["EDSS"]
+        else:
+            if "EDSS" not in parsed:
+                print("⚠️   'klassifizierbar' is true but EDSS missing — adding fallback.")
+                parsed["EDSS"] = 7.0
+
+        inference_time = time.time() - start_time
+
+        return {
+            "success": True,
+            "result": parsed,
+            "inference_time_sec": inference_time
+        }
+
+    except Exception as e:
+        print(f"❌ Inference error: {e}")
+        return {
+            "success": False,
+            "error": str(e),
+            "inference_time_sec": -1,
+            "result": None
+        }
+
+# === BUILD PATIENT TEXT ===
+def build_patient_text(row):
+    return (
+        str(row.get("T_Zusammenfassung", "")) + "\n" +
+        str(row.get("Diagnosen", "")) + "\n" +
+        str(row.get("T_KlinBef", "")) + "\n" +
+        str(row.get("T_Befunde", ""))
+    )
+
+# === MAIN LOOP (NEW: MULTI-ITERATION) ===
+if __name__ == "__main__":
+    # Load data ONCE (to avoid repeated I/O overhead)
+    df = pd.read_csv(INPUT_CSV, sep=';')
+    total_rows = len(df)
+    print(f"Loaded {total_rows} patient records.")
+
+    for iteration in range(1, NUM_ITERATIONS + 1):
+        print(f"\n{'='*60}")
+        print(f"🔄 ITERATION {iteration}/{NUM_ITERATIONS}")
+        print(f"{'='*60}")
+
+        iteration_results = []
+        start_iter = time.time()
+
+        for idx, row in df.iterrows():
+            print(f"\rRow {idx+1}/{total_rows} | Iter {iteration}", end='', flush=True)
+            try:
+                patient_text = build_patient_text(row)
+                result = run_inference(patient_text)
+
+                # Attach metadata
+                if result["success"]:
+                    res = result["result"].copy()  # avoid mutation
+                    res["iteration"] = iteration
+                    res["unique_id"] = row.get("unique_id", f"row_{idx}")
+                    res["MedDatum"] = row.get("MedDatum", None)
+                    result["result"] = res
+
+                else:
+                    result["iteration"] = iteration
+                    result["unique_id"] = row.get("unique_id", f"row_{idx}")
+                    result["MedDatum"] = row.get("MedDatum", None)
+
+                iteration_results.append(result)
+
+                if result["success"]:
+                    res = result["result"]
+                    edss = res.get("EDSS", "N/A") if res.get("klassifizierbar") else "N/A"
+                    print(f" ✅ EDSS={edss}, cert={res.get('certainty_percent', '?')}%")
+                else:
+                    print(f" ❌ {result.get('error', 'Unknown')}")
+
+            except Exception as e:
+                print(f"\n⚠️   Row {idx} failed: {e}")
+                iteration_results.append({
+                    "success": False,
+                    "error": str(e),
+                    "iteration": iteration,
+                    "unique_id": row.get("unique_id", f"row_{idx}"),
+                    "MedDatum": row.get("MedDatum", None),
+                    "result": None
+                })
+                if STOP_ON_FIRST_ERROR:
+                    break
+
+        # Save per-iteration results
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        output_path = INPUT_CSV.replace(".csv", f"_results_iter_{iteration}_{timestamp}.json")
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(iteration_results, f, indent=2, ensure_ascii=False)
+        print(f"\n✅ Iteration {iteration} complete. Saved to: {output_path}")
+
+        elapsed = time.time() - start_iter
+        print(f"⏱️   Iteration {iteration} took {elapsed:.1f}s ({elapsed/total_rows:.1f}s/row)")
+
+    print(f"\n🎉 All {NUM_ITERATIONS} iterations completed!")
+
+
+
+##

figure1.py

@@ -320,3 +320,63 @@ plt.tight_layout()
 plt.show()
 
 ##
+
+# %% Patientjourney Bubble chart
+import matplotlib.pyplot as plt
+import numpy as np
+
+import matplotlib as mpl
+
+mpl.rcParams["font.family"] = "DejaVu Sans"   # or "Arial", "Calibri", "Times New Roman", ...
+mpl.rcParams["font.size"] = 12                # default size for text
+mpl.rcParams["axes.titlesize"] = 14
+mpl.rcParams["axes.titleweight"] = "bold"
+
+
+# Data (your counts)
+visits = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])
+patient_count = np.array([32, 24, 28, 17, 13, 6, 3, 3, 2])
+
+# "Remaining" = patients with >= that many visits (cumulative from the right)
+remaining = np.array([patient_count[i:].sum() for i in range(len(patient_count))])
+
+# --- Plot ---
+fig, ax = plt.subplots(figsize=(12, 3))
+
+y = 0.0  # all bubbles on one horizontal line
+
+# Horizontal line
+ax.hlines(y, visits.min() - 0.4, visits.max() + 0.4, color="#1f77b4", linewidth=3)
+
+# Bubble sizes (scale as needed)
+# (Matplotlib scatter uses area in points^2)
+sizes = patient_count * 35  # tweak this multiplier if you want bigger/smaller bubbles
+
+ax.scatter(visits, np.full_like(visits, y), s=sizes, color="#1f77b4", zorder=3)
+
+# Title
+#ax.set_title("Patient Journey by Visit Count", fontsize=14, pad=18)
+
+# Top labels: "1 visits", "2 visits", ...
+for x in visits:
+    label = f"{x} visit" if x == 1 else f"{x} visits"
+    ax.text(x, y + 0.18, label, ha="center", va="bottom", fontsize=10)
+
+# Bottom labels: "X patients" and "Y remaining"
+for x, pc, rem in zip(visits, patient_count, remaining):
+    ax.text(x, y - 0.20, f"{pc} patients", ha="center", va="top", fontsize=9)
+    ax.text(x, y - 0.32, f"{rem} remaining", ha="center", va="top", fontsize=9)
+
+# Cosmetics: remove axes, keep spacing nice
+ax.set_xlim(visits.min() - 0.6, visits.max() + 0.6)
+ax.set_ylim(-0.5, 0.35)
+ax.set_xticks([])
+ax.set_yticks([])
+for spine in ax.spines.values():
+    spine.set_visible(False)
+
+plt.tight_layout()
+plt.show()
+plt.savefig("patient_journey.svg", format="svg", bbox_inches="tight")
+##
+

show_plots.py

@@ -401,7 +401,7 @@ import seaborn as sns
 import numpy as np
 
 # Load your data from TSV file
-file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/join_results_unique.tsv'
+file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/results/join_results_unique.tsv'
 df = pd.read_csv(file_path, sep='\t')
 
 # Replace comma with dot for numeric conversion in GT.EDSS and result.EDSS
@@ -718,128 +718,325 @@ plt.show()
 
 ##
 
-
+# %% Dashboard
-
-
-
-
-# %% Dashboard 
 import pandas as pd
 import matplotlib.pyplot as plt
-import seaborn as sns
+import matplotlib.dates as mdates
-from datetime import datetime
 import numpy as np
+from matplotlib.gridspec import GridSpec
+
+def to_numeric_comma(s: pd.Series) -> pd.Series:
+    # accepts 1.5 and 1,5
+    return pd.to_numeric(s.astype(str).str.replace(",", ".", regex=False), errors="coerce")
 
 # Load the data
 file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv'
 df = pd.read_csv(file_path, sep='\t')
 
-# Rename columns to remove 'result.' prefix and handle spaces
+# Rename columns to remove 'result.' prefix and replace spaces
 column_mapping = {}
 for col in df.columns:
     if col.startswith('result.'):
-        new_name = col.replace('result.', '')
+        new_name = col.replace('result.', '').replace(' ', '_')
-        # Handle spaces in column names (replace with underscores if needed)
-        new_name = new_name.replace(' ', '_')
         column_mapping[col] = new_name
 df = df.rename(columns=column_mapping)
 
-# Convert MedDatum to datetime
+# Parse MedDatum safely
-df['MedDatum'] = pd.to_datetime(df['MedDatum'])
+df['MedDatum'] = pd.to_datetime(df['MedDatum'], errors='coerce')
 
-# Check what columns actually exist in the dataset
+# Patient
-print("Available columns:")
+patient_id = 'd13e4aa3'
-print(df.columns.tolist())
+patient_data = df[df['unique_id'] == patient_id].sort_values('MedDatum').copy()
-print("\nFirst few rows:")
+if patient_data.empty:
-print(df.head())
+    raise ValueError(f"No data found for patient: {patient_id}")
 
-# Hardcode specific patient names
+# Functional systems + EDSS
-patient_names = ['6b56865d']
+edss_col, edss_title = ('GT.EDSS', 'EDSS')
 
-# Define the functional systems (columns to plot) - adjust based on actual column names
+functional_systems = [
-functional_systems = ['EDSS', 'Visual', 'Sensory', 'Motor', 'Brainstem', 'Cerebellar', 'Autonomic', 'Bladder', 'Intellectual']
+    ('GT.VISUAL_OPTIC_FUNCTIONS', 'Visual / Optic'),
+    ('GT.CEREBELLAR_FUNCTIONS', 'Cerebellar'),
+    ('GT.BRAINSTEM_FUNCTIONS', 'Brainstem'),
+    ('GT.SENSORY_FUNCTIONS', 'Sensory'),
+    ('GT.PYRAMIDAL_FUNCTIONS', 'Pyramidal (Motor)'),
+    ('GT.AMBULATION', 'Ambulation'),
+    ('GT.CEREBRAL_FUNCTIONS', 'Cerebral'),
+    ('GT.BOWEL_AND_BLADDER_FUNCTIONS', 'Bowel & Bladder'),
+]
 
-# Create subplots horizontally (2 columns, adjust rows as needed)
+# y-axis max rules
-num_plots = len(functional_systems)
+ymax_by_col = {
-num_cols = 2
+    'GT.PYRAMIDAL_FUNCTIONS': 6,
-num_rows = (num_plots + num_cols - 1) // num_cols  # Ceiling division
+    'GT.SENSORY_FUNCTIONS': 6,
+    'GT.BOWEL_AND_BLADDER_FUNCTIONS': 6,
+    'GT.VISUAL_OPTIC_FUNCTIONS': 6,
+    'GT.CEREBELLAR_FUNCTIONS': 5,
+    'GT.CEREBRAL_FUNCTIONS': 5,
+    'GT.BRAINSTEM_FUNCTIONS': 5,
+    'GT.EDSS': 10,
+}
+default_ymax = 6
 
-fig, axes = plt.subplots(num_rows, num_cols, figsize=(15, 4*num_rows), sharex=False)  # Changed sharex=False
+# ---------- Build shared "event dates" ticks ----------
-if num_plots == 1:
+cols_for_dates = [edss_col] + [c for c, _ in functional_systems]
-    axes = [axes]
+event_dates = []
-elif num_rows == 1:
-    axes = axes
-else:
-    axes = axes.flatten()
 
-# Plot for the hardcoded patient
+for c in cols_for_dates:
-for i, system in enumerate(functional_systems):
+    if c in patient_data.columns:
-    # Filter data for this specific patient
+        y = to_numeric_comma(patient_data[c])  # <-- changed
-    patient_data = df[df['unique_id'] == patient_names[0]].sort_values('MedDatum')
+        x = patient_data['MedDatum']
+        tmp = pd.DataFrame({"x": x, "y": y}).dropna(subset=["x", "y"])
+        event_dates.extend(tmp["x"].tolist())
 
-    # Check if patient data exists
+event_dates = sorted(pd.Series(event_dates).drop_duplicates().tolist())
-    if patient_data.empty:
-        print(f"No data found for patient: {patient_names[0]}")
-        continue
 
-    # Check if the system column exists in the data
+max_ticks = 8
-    if system in patient_data.columns:
+if len(event_dates) > max_ticks:
-        # Plot the specific functional system
+    idx = np.linspace(0, len(event_dates) - 1, max_ticks, dtype=int)
-        if not patient_data[system].isna().all():
+    event_dates = [event_dates[i] for i in idx]
-            axes[i].plot(patient_data['MedDatum'], patient_data[system], marker='o', linewidth=2, label=system)
+
-            axes[i].set_ylabel('Score')
+# ---------- A4 figure ----------
-            axes[i].set_title(f'Functional System: {system}')
+fig = plt.figure(figsize=(11.69, 8.27))
-            axes[i].grid(True, alpha=0.3)
+gs = GridSpec(nrows=3, ncols=4, figure=fig, height_ratios=[2.0, 1.0, 1.0], hspace=0.5, wspace=0.35)
-            axes[i].legend()
+
-        else:
+def style_time_axis(ax, show_labels=True):
-            axes[i].set_title(f'Functional System: {system} (No data)')
+    ax.set_xticks(event_dates)
-            axes[i].set_ylabel('Score')
+    ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))
-            axes[i].grid(True, alpha=0.3)
+    ax.tick_params(axis='x', rotation=30, labelsize=8, pad=2)
+    if not show_labels:
+        ax.tick_params(labelbottom=False)
+
+# ---------- EDSS main plot ----------
+ax_main = fig.add_subplot(gs[0, :])
+
+if edss_col in patient_data.columns:
+    y = to_numeric_comma(patient_data[edss_col])  # <-- changed
+    x = patient_data['MedDatum']
+    plot_df = pd.DataFrame({"x": x, "y": y}).dropna(subset=["x", "y"]).sort_values("x")
+
+    ax_main.set_title(edss_title, fontsize=14, fontweight='bold')
+    ax_main.set_ylabel("Score")
+    ax_main.set_ylim(0, ymax_by_col.get(edss_col, default_ymax))
+    ax_main.grid(True, alpha=0.3)
+
+    if not plot_df.empty:
+        ax_main.plot(plot_df["x"], plot_df["y"], marker='o', linewidth=3, color='tab:red')
     else:
-        # Try to find column with similar name (case insensitive)
+        ax_main.set_title("EDSS (no numeric data)", fontsize=14, fontweight='bold')
-        found_column = None
+else:
-        for col in df.columns:
+    ax_main.set_title("EDSS (missing column GT.EDSS)", fontsize=14, fontweight='bold')
-            if system.lower() in col.lower():
+    ax_main.set_ylim(0, ymax_by_col.get(edss_col, 10))
-                found_column = col
+    ax_main.grid(True, alpha=0.3)
-                break
 
-        if found_column:
+style_time_axis(ax_main)
-            print(f"Found similar column: {found_column}")
+
-            if not patient_data[found_column].isna().all():
+# ---------- Small aligned plots ----------
-                axes[i].plot(patient_data['MedDatum'], patient_data[found_column], marker='o', linewidth=2, label=found_column)
+small_axes = []
-                axes[i].set_ylabel('Score')
+for k, (col, title) in enumerate(functional_systems):
-                axes[i].set_title(f'Functional System: {system} (found as: {found_column})')
+    r = 1 + (k // 4)
-                axes[i].grid(True, alpha=0.3)
+    c = (k % 4)
-                axes[i].legend()
+    ax = fig.add_subplot(gs[r, c], sharex=ax_main)
+    small_axes.append(ax)
+
+    ymax = ymax_by_col.get(col, default_ymax)
+    ax.set_title(title, fontsize=10)
+    ax.set_ylabel("Score")
+    ax.set_ylim(0, ymax)
+    ax.grid(True, alpha=0.3)
+
+    if col in patient_data.columns:
+        y = to_numeric_comma(patient_data[col])  # <-- changed
+        x = patient_data['MedDatum']
+        plot_df = pd.DataFrame({"x": x, "y": y}).dropna(subset=["x", "y"]).sort_values("x")
+
+        if not plot_df.empty:
+            ax.plot(plot_df["x"], plot_df["y"], marker='o', linewidth=2, color='tab:blue')
         else:
-            axes[i].set_title(f'Functional System: {system} (Column not found)')
+            ax.set_title(f"{title} (no data)", fontsize=10)
-            axes[i].set_ylabel('Score')
+    else:
-            axes[i].grid(True, alpha=0.3)
+        ax.set_title(f"{title} (missing)", fontsize=10)
 
-# Hide empty subplots
+    style_time_axis(ax)
-for i in range(len(functional_systems), len(axes)):
-    axes[i].set_visible(False)
 
-# Set x-axis label for the last row only
+# Hide x tick labels on first row of small plots
-for i in range(len(functional_systems)):
+for ax in small_axes[:4]:
-    if i >= len(axes) - num_cols:  # Last row
+    ax.tick_params(labelbottom=False)
-        axes[i].set_xlabel('Date')
-
-# Force date formatting on all axes
-for ax in axes:
-    ax.tick_params(axis='x', rotation=45)
-    ax.xaxis.set_major_formatter(plt.matplotlib.dates.DateFormatter('%Y-%m-%d'))
-    ax.xaxis.set_major_locator(plt.matplotlib.dates.MonthLocator())
-
-# Automatically format x-axis dates
-plt.gcf().autofmt_xdate()
 
 plt.tight_layout()
+fig.subplots_adjust(hspace=0.7)
 plt.show()
+##
+
+<<<<<<< Updated upstream
+=======
+
+
+
+# %% Dashboard Angepasst
+import pandas as pd
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+import numpy as np
+from matplotlib.gridspec import GridSpec
+
+def to_numeric_comma(s: pd.Series) -> pd.Series:
+    # accepts 1.5 and 1,5
+    return pd.to_numeric(s.astype(str).str.replace(",", ".", regex=False), errors="coerce")
+
+# Load the data
+file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv'
+df = pd.read_csv(file_path, sep='\t')
+
+# Rename columns to remove 'result.' prefix and replace spaces
+column_mapping = {}
+for col in df.columns:
+    if col.startswith('result.'):
+        new_name = col.replace('result.', '').replace(' ', '_')
+        column_mapping[col] = new_name
+df = df.rename(columns=column_mapping)
+
+# Parse MedDatum safely
+df['MedDatum'] = pd.to_datetime(df['MedDatum'], errors='coerce')
+
+# Patient
+patient_id = '3d942c60'
+
+patient_data = df[df['unique_id'] == patient_id].sort_values('MedDatum').copy()
+if patient_data.empty:
+    raise ValueError(f"No data found for patient: {patient_id}")
+
+# Functional systems + EDSS
+edss_col, edss_title = ('GT.EDSS', 'EDSS')
+
+functional_systems = [
+    ('GT.VISUAL_OPTIC_FUNCTIONS', 'Visual / Optic'),
+    ('GT.CEREBELLAR_FUNCTIONS', 'Cerebellar'),
+    ('GT.BRAINSTEM_FUNCTIONS', 'Brainstem'),
+    ('GT.SENSORY_FUNCTIONS', 'Sensory'),
+    ('GT.PYRAMIDAL_FUNCTIONS', 'Pyramidal (Motor)'),
+    ('GT.AMBULATION', 'Ambulation'),
+    ('GT.CEREBRAL_FUNCTIONS', 'Cerebral'),
+    ('GT.BOWEL_AND_BLADDER_FUNCTIONS', 'Bowel & Bladder'),
+]
+
+# y-axis max rules
+ymax_by_col = {
+    'GT.PYRAMIDAL_FUNCTIONS': 6,
+    'GT.SENSORY_FUNCTIONS': 6,
+    'GT.BOWEL_AND_BLADDER_FUNCTIONS': 6,
+    'GT.VISUAL_OPTIC_FUNCTIONS': 6,
+    'GT.CEREBELLAR_FUNCTIONS': 5,
+    'GT.CEREBRAL_FUNCTIONS': 5,
+    'GT.BRAINSTEM_FUNCTIONS': 5,
+    'GT.EDSS': 10,
+}
+default_ymax = 6
+
+# ---------- Build shared visit dates ticks ----------
+# Use ALL patient visit dates, not only dates with valid numeric values
+event_dates = sorted(patient_data['MedDatum'].dropna().drop_duplicates().tolist())
+
+max_ticks = 8
+if len(event_dates) > max_ticks:
+    idx = np.linspace(0, len(event_dates) - 1, max_ticks, dtype=int)
+    event_dates = [event_dates[i] for i in idx]
+
+# Base timeline for plotting: one row per patient visit date
+timeline = (
+    patient_data[['MedDatum']]
+    .dropna()
+    .drop_duplicates()
+    .sort_values('MedDatum')
+    .rename(columns={'MedDatum': 'x'})
+)
+
+# ---------- A4 figure ----------
+fig = plt.figure(figsize=(11.69, 8.27))
+gs = GridSpec(nrows=3, ncols=4, figure=fig, height_ratios=[2.0, 1.0, 1.0], hspace=0.5, wspace=0.35)
+
+def style_time_axis(ax, show_labels=True):
+    ax.set_xticks(event_dates)
+    ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))
+    ax.tick_params(axis='x', rotation=30, labelsize=8, pad=2)
+    if not show_labels:
+        ax.tick_params(labelbottom=False)
+
+def get_plot_df(patient_data, col):
+    """
+    Keep all visit dates.
+    Missing values stay NaN so matplotlib draws gaps instead of zeros.
+    """
+    tmp = patient_data[['MedDatum', col]].copy()
+    tmp = tmp.rename(columns={'MedDatum': 'x', col: 'raw_y'})
+    tmp['y'] = to_numeric_comma(tmp['raw_y'])
+
+    # aggregate if multiple rows exist on same date
+    tmp = tmp.groupby('x', as_index=False)['y'].max()
+
+    # merge onto full timeline so all dates remain visible
+    plot_df = timeline.merge(tmp, on='x', how='left').sort_values('x')
+    return plot_df
+
+# ---------- EDSS main plot ----------
+ax_main = fig.add_subplot(gs[0, :])
+ax_main.set_title(edss_title, fontsize=14, fontweight='bold')
+ax_main.set_ylabel("Score")
+ax_main.set_ylim(0, ymax_by_col.get(edss_col, default_ymax))
+ax_main.grid(True, alpha=0.3)
+
+if edss_col in patient_data.columns:
+    plot_df = get_plot_df(patient_data, edss_col)
+
+    if plot_df['y'].notna().any():
+        # NaNs create visible gaps in the line
+        ax_main.plot(plot_df["x"], plot_df["y"], marker='o', linewidth=3, color='tab:red')
+    else:
+        ax_main.set_title("EDSS (no numeric data)", fontsize=14, fontweight='bold')
+else:
+    ax_main.set_title("EDSS (missing column GT.EDSS)", fontsize=14, fontweight='bold')
+
+style_time_axis(ax_main)
+
+# ---------- Small aligned plots ----------
+small_axes = []
+for k, (col, title) in enumerate(functional_systems):
+    r = 1 + (k // 4)
+    c = (k % 4)
+    ax = fig.add_subplot(gs[r, c], sharex=ax_main)
+    small_axes.append(ax)
+
+    ymax = ymax_by_col.get(col, default_ymax)
+    ax.set_title(title, fontsize=10)
+    ax.set_ylabel("Score")
+    ax.set_ylim(0, ymax)
+    ax.grid(True, alpha=0.3)
+
+    if col in patient_data.columns:
+        plot_df = get_plot_df(patient_data, col)
+
+        if plot_df['y'].notna().any():
+            # NaNs remain in y -> line breaks where data is missing
+            ax.plot(plot_df["x"], plot_df["y"], marker='o', linewidth=2, color='tab:blue')
+        else:
+            ax.set_title(f"{title} (no numeric data)", fontsize=10)
+    else:
+        ax.set_title(f"{title} (missing)", fontsize=10)
+
+    style_time_axis(ax)
+
+# Hide x tick labels on first row of small plots
+for ax in small_axes[:4]:
+    ax.tick_params(labelbottom=False)
+
+plt.tight_layout()
+fig.subplots_adjust(hspace=0.7)
+plt.show()
+
+
 
 ##
 
-
+>>>>>>> Stashed changes
 # %% Table 
 import pandas as pd
 import matplotlib.pyplot as plt
@@ -1737,100 +1934,7 @@ plt.show()
 
 
 
-# %% Difference Gemini easy
+# %% Difference Plot Gemini 
-
-
-# --- 1. Process Error Data ---
-system_names = [name.split('.')[1] for name, _ in functional_systems_to_plot]
-plot_list = []
-
-for gt_col, res_col in functional_systems_to_plot:
-    sys_name = gt_col.split('.')[1]
-
-    # Robust parsing
-    gt = df[gt_col].apply(safe_parse)
-    res = df[res_col].apply(safe_parse)
-    error = res - gt
-
-    # Calculate counts
-    matches = (error == 0).sum()
-    under = (error < 0).sum()
-    over = (error > 0).sum()
-    total = error.dropna().count()
-
-    # Calculate Percentages
-    # Using max(total, 1) to avoid division by zero
-    divisor = max(total, 1)
-    match_pct = (matches / divisor) * 100
-    under_pct = (under / divisor) * 100
-    over_pct = (over / divisor) * 100
-
-    plot_list.append({
-        'System': sys_name.replace('_', ' ').title(),
-        'Matches': matches,
-        'MatchPct': match_pct,
-        'Under': under,
-        'UnderPct': under_pct,
-        'Over': over,
-        'OverPct': over_pct
-    })
-
-stats_df = pd.DataFrame(plot_list)
-
-# --- 2. Plotting ---
-fig, ax = plt.subplots(figsize=(12, 8)) # Slightly taller for multi-line labels
-
-color_under = '#E74C3C'
-color_over = '#3498DB'
-bar_height = 0.6
-
-y_pos = np.arange(len(stats_df))
-
-ax.barh(y_pos, -stats_df['Under'], bar_height, label='Under-scored', color=color_under, edgecolor='white', alpha=0.8)
-ax.barh(y_pos, stats_df['Over'], bar_height, label='Over-scored', color=color_over, edgecolor='white', alpha=0.8)
-
-# --- 3. Aesthetics & Labels ---
-
-for i, row in stats_df.iterrows():
-    # Constructing a detailed label for the left side
-    # Matches (Bold) | Under % | Over %
-    label_text = (
-        f"$\mathbf{{{row['System']}}}$\n"
-        f"Matches: {int(row['Matches'])} ({row['MatchPct']:.1f}%)\n"
-        f"Under: {int(row['Under'])} ({row['UnderPct']:.1f}%) | Over: {int(row['Over'])} ({row['OverPct']:.1f}%)"
-    )
-
-    # Position text to the left of the x=0 line
-    ax.text(ax.get_xlim()[0] - 0.5, i, label_text, va='center', ha='right', fontsize=9, color='#333333', linespacing=1.3)
-
-# Zero line
-ax.axvline(0, color='black', linewidth=1.2, alpha=0.7)
-
-# Clean up axes
-ax.set_yticks([])
-ax.set_xlabel('Number of Patients with Error', fontsize=11, fontweight='bold', labelpad=10)
-#ax.set_title('Directional Error Analysis by Functional System', fontsize=14, pad=30)
-
-# Make X-axis labels absolute
-ax.set_xticklabels([int(abs(tick)) for tick in ax.get_xticks()])
-
-# Remove spines
-for spine in ['top', 'right', 'left']:
-    ax.spines[spine].set_visible(False)
-
-# Legend
-ax.legend(loc='upper right', frameon=False, bbox_to_anchor=(1, 1.1))
-
-# Grid
-ax.xaxis.grid(True, linestyle='--', alpha=0.3)
-
-plt.tight_layout()
-plt.show()
-##
-
-
-
-# %% name
 import pandas as pd
 import matplotlib.pyplot as plt
 import os
@@ -1919,6 +2023,287 @@ ax.legend(loc='upper right', frameon=False, bbox_to_anchor=(1, 1.1), ncol=2)
 plt.tight_layout()
 plt.show()
 ##
+
+
+# %% Functional System Error Boxplots
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+import numpy as np
+from matplotlib.patches import Patch
+from matplotlib.lines import Line2D
+
+# --- Configuration & Theme ---
+plt.rcParams['font.family'] = 'Arial'
+figure_save_path = 'project/visuals/functional_systems_boxplot.svg'
+
+# --- 1. Build error data for boxplots ---
+boxplot_data = []
+system_labels = []
+sample_sizes = []
+
+for gt_col, res_col in functional_systems_to_plot:
+    sys_name = gt_col.split('.')[1]
+
+    # Robust parsing
+    gt = df[gt_col].apply(safe_parse)
+    res = df[res_col].apply(safe_parse)
+
+    # Error = result - ground truth
+    error = (res - gt).dropna()
+
+    # Ignore all 0 errors
+    error = error[error != 0]
+
+    # Keep only systems that actually have non-zero data
+    if len(error) > 0:
+        clean_name = sys_name.replace('_', ' ').title()
+        boxplot_data.append(error.values)
+        system_labels.append(clean_name)
+        sample_sizes.append(len(error))
+
+# Safety check
+if not boxplot_data:
+    raise ValueError("No valid non-zero error data available for any functional system.")
+
+# Put n into x-axis labels so it doesn't overlap the plot
+xtick_labels = [f"{label}\n(n={n})" for label, n in zip(system_labels, sample_sizes)]
+
+# --- 2. Plotting ---
+fig, ax = plt.subplots(figsize=(14, 8))
+
+bp = ax.boxplot(
+    boxplot_data,
+    vert=True,
+    patch_artist=True,
+    labels=xtick_labels,
+    showmeans=True,
+    meanline=False
+)
+
+# --- 3. Styling ---
+box_face = '#D6EAF8'
+box_edge = '#2980B9'
+whisker_col = '#7F8C8D'
+median_col = '#C0392B'
+mean_col = '#1ABC9C'
+flier_face = '#95A5A6'
+flier_edge = '#7F8C8D'
+
+for box in bp['boxes']:
+    box.set(facecolor=box_face, edgecolor=box_edge, linewidth=1.5)
+
+for whisker in bp['whiskers']:
+    whisker.set(color=whisker_col, linewidth=1.2)
+
+for cap in bp['caps']:
+    cap.set(color=whisker_col, linewidth=1.2)
+
+for median in bp['medians']:
+    median.set(color=median_col, linewidth=2)
+
+for mean in bp['means']:
+    mean.set(marker='o', markerfacecolor=mean_col, markeredgecolor='black', markersize=6)
+
+for flier in bp['fliers']:
+    flier.set(marker='o', markerfacecolor=flier_face, markeredgecolor=flier_edge, alpha=0.6, markersize=4)
+
+# Reference line at zero error
+ax.axhline(0, color='black', linewidth=1.2, linestyle='--')
+
+# Labels and formatting
+ax.set_xlabel('Functional System', fontsize=11, fontweight='bold')
+ax.set_ylabel('Error (Result - Ground Truth)', fontsize=11, fontweight='bold')
+
+# Rotate x labels for readability
+plt.xticks(rotation=45, ha='right')
+
+# Grid and spines
+ax.yaxis.grid(True, linestyle='--', alpha=0.3)
+for spine in ['top', 'right']:
+    ax.spines[spine].set_visible(False)
+
+# --- 4. Legend above the plot, outside the axes ---
+legend_handles = [
+    Patch(facecolor=box_face, edgecolor=box_edge, label='IQR (25th-75th percentile)'),
+    Line2D([0], [0], color=median_col, lw=2, label='Median'),
+    Line2D([0], [0], marker='o', color='w', markerfacecolor=mean_col,
+           markeredgecolor='black', markersize=7, label='Mean'),
+    Line2D([0], [0], marker='o', color='w', markerfacecolor=flier_face,
+           markeredgecolor=flier_edge, alpha=0.8, markersize=6, label='Outlier'),
+    Line2D([0], [0], color='black', lw=1.2, linestyle='--', label='Zero error reference')
+]
+
+ax.legend(
+    handles=legend_handles,
+    loc='lower center',
+    bbox_to_anchor=(0.5, 1.02),
+    ncol=3,
+    frameon=False
+)
+
+# Leave room at the top for the legend
+plt.tight_layout(rect=[0, 0, 1, 0.90])
+
+# Optional save
+os.makedirs(os.path.dirname(figure_save_path), exist_ok=True)
+plt.savefig(figure_save_path, format='svg', bbox_inches='tight')
+
+plt.show()
+##
+
+<<<<<<< Updated upstream
+=======
+# %% Functional System + EDSS Error Boxplots
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+import numpy as np
+from matplotlib.patches import Patch
+from matplotlib.lines import Line2D
+
+# --- Configuration & Theme ---
+plt.rcParams['font.family'] = 'Arial'
+figure_save_path = 'project/visuals/functional_systems_edss_boxplot.svg'
+
+# ------------------------------------------------------------
+# Expect functional_systems_to_plot like:
+# [
+#   ('GT.VISUAL_OPTIC_FUNCTIONS', 'result.VISUAL_OPTIC_FUNCTIONS'),
+#   ...
+# ]
+#
+# Add EDSS here:
+# ------------------------------------------------------------
+all_systems_to_plot = list(functional_systems_to_plot) + [
+    ('GT.EDSS', 'result.EDSS')
+]
+
+# --- 1. Build error data for boxplots ---
+boxplot_data = []
+system_labels = []
+sample_sizes = []
+
+for gt_col, res_col in all_systems_to_plot:
+    # Skip safely if a column is missing
+    if gt_col not in df.columns or res_col not in df.columns:
+        print(f"Skipping missing columns: {gt_col}, {res_col}")
+        continue
+
+    sys_name = gt_col.split('.')[1]
+
+    # Robust parsing
+    gt = df[gt_col].apply(safe_parse)
+    res = df[res_col].apply(safe_parse)
+
+    # Error = result - ground truth
+    error = (res - gt).dropna()
+
+    # Ignore all 0 errors
+    error = error[error != 0]
+
+    # Keep only systems that actually have non-zero data
+    if len(error) > 0:
+        if sys_name == 'EDSS':
+            clean_name = 'EDSS'
+        else:
+            clean_name = sys_name.replace('_', ' ').title()
+
+        boxplot_data.append(error.values)
+        system_labels.append(clean_name)
+        sample_sizes.append(len(error))
+
+# Safety check
+if not boxplot_data:
+    raise ValueError("No valid non-zero error data available for any functional system or EDSS.")
+
+# Put n into x-axis labels so it doesn't overlap the plot
+xtick_labels = [f"{label}\n(n={n})" for label, n in zip(system_labels, sample_sizes)]
+
+# --- 2. Plotting ---
+fig, ax = plt.subplots(figsize=(15, 8))
+
+bp = ax.boxplot(
+    boxplot_data,
+    vert=True,
+    patch_artist=True,
+    labels=xtick_labels,
+    showmeans=True,
+    meanline=False
+)
+
+# --- 3. Styling ---
+box_face = '#D6EAF8'
+box_edge = '#2980B9'
+whisker_col = '#7F8C8D'
+median_col = '#C0392B'
+mean_col = '#1ABC9C'
+flier_face = '#95A5A6'
+flier_edge = '#7F8C8D'
+
+for box in bp['boxes']:
+    box.set(facecolor=box_face, edgecolor=box_edge, linewidth=1.5)
+
+for whisker in bp['whiskers']:
+    whisker.set(color=whisker_col, linewidth=1.2)
+
+for cap in bp['caps']:
+    cap.set(color=whisker_col, linewidth=1.2)
+
+for median in bp['medians']:
+    median.set(color=median_col, linewidth=2)
+
+for mean in bp['means']:
+    mean.set(marker='o', markerfacecolor=mean_col, markeredgecolor='black', markersize=6)
+
+for flier in bp['fliers']:
+    flier.set(marker='o', markerfacecolor=flier_face, markeredgecolor=flier_edge, alpha=0.6, markersize=4)
+
+# Reference line at zero error
+ax.axhline(0, color='black', linewidth=1.2, linestyle='--')
+
+# Labels and formatting
+ax.set_xlabel('Functional System / EDSS', fontsize=11, fontweight='bold')
+ax.set_ylabel('Error (Result - Ground Truth)', fontsize=11, fontweight='bold')
+
+# Rotate x labels for readability
+plt.xticks(rotation=45, ha='right')
+
+# Grid and spines
+ax.yaxis.grid(True, linestyle='--', alpha=0.3)
+for spine in ['top', 'right']:
+    ax.spines[spine].set_visible(False)
+
+# --- 4. Legend above the plot, outside the axes ---
+legend_handles = [
+    Patch(facecolor=box_face, edgecolor=box_edge, label='IQR (25th-75th percentile)'),
+    Line2D([0], [0], color=median_col, lw=2, label='Median'),
+    Line2D([0], [0], marker='o', color='w', markerfacecolor=mean_col,
+           markeredgecolor='black', markersize=7, label='Mean'),
+    Line2D([0], [0], marker='o', color='w', markerfacecolor=flier_face,
+           markeredgecolor=flier_edge, alpha=0.8, markersize=6, label='Outlier'),
+    Line2D([0], [0], color='black', lw=1.2, linestyle='--', label='Zero error reference')
+]
+
+ax.legend(
+    handles=legend_handles,
+    loc='lower center',
+    bbox_to_anchor=(0.5, 1.02),
+    ncol=3,
+    frameon=False
+)
+
+# Leave room at the top for the legend
+plt.tight_layout(rect=[0, 0, 1, 0.90])
+
+# Optional save
+os.makedirs(os.path.dirname(figure_save_path), exist_ok=True)
+plt.savefig(figure_save_path, format='svg', bbox_inches='tight')
+
+plt.show()
+##
+>>>>>>> Stashed changes
+
 # %% test
 # Diagnose: what are the actual differences?
 print("\n🔍 Raw differences (first 5 rows per system):")