Adjsuting and cleaning

Directional Errors of each functional system.

.gitignore

@@ -1,7 +1,20 @@
-# Ignore all contents of these directories
-!**/*.py
-/Data/
-/attach/
-/results/
-/enarcelona/
+# 1. Broad Ignores
+/Data/*
+/attach/*
+/results/*
+/enarcelona/*
 .env
+__pycache__/
+*.pyc
+
+=======
+/reference/
+*.svg
+>>>>>>> Stashed changes
+# 2. Ignore virtual environments COMPLETELY
+# This must come BEFORE the unignore rule
+env*/
+
+# 3. The "Unignore" rule (Whitelisting)
+# We only unignore .py files that aren't already blocked by the rules above
+!**/*.py

app.py

@@ -214,3 +214,8 @@ if __name__ == "__main__":
     print(f"Results saved to {output_json}")
 ##
 
+
+
+# %% name
+eXXXXXXXX
+##

figure1.py

@@ -263,3 +263,60 @@ plt.legend(frameon=False, loc='upper center', bbox_to_anchor=(0.5, -0.05))
 plt.tight_layout()
 plt.show()
 ##
+
+
+
+
+# %% name
+import matplotlib.pyplot as plt
+
+# Data
+data = {
+    'Visit': [9, 8, 7, 6, 5, 4, 3, 2, 1],
+    'patient_count': [2, 3, 3, 6, 13, 17, 28, 24, 32]
+}
+
+# Create figure and axis
+fig, ax = plt.subplots(figsize=(10, 6))
+
+# Plot the bar chart
+bars = ax.bar(data['Visit'], data['patient_count'], color='darkblue', label='Patients by Visit Count')
+
+# Add labels and title
+ax.set_xlabel('Visit Number (from last to first)', fontsize=12)
+ax.set_ylabel('Number of Patients', fontsize=12)
+ax.set_title('Patient Visits by Visit Number', fontsize=14)
+
+# Invert x-axis to show Visit 9 on the left (descending order) if desired, but keep natural order (1–9 left to right)
+# For descending order (9→1 from left to right), we'd need to reverse:
+# Visit = data['Visit'][::-1], patient_count = data['patient_count'][::-1]
+# But standard practice is ascending (1 to 9), so we'll sort accordingly:
+# Let's sort by Visit to ensure left-to-right: 1,2,...,9
+
+# Actually, your current Visit list is [9,8,...,1], which is descending.
+# Let's sort by Visit for intuitive left-to-right increasing order:
+sorted_indices = sorted(range(len(data['Visit'])), key=lambda i: data['Visit'][i])
+visit_sorted = [data['Visit'][i] for i in sorted_indices]
+count_sorted = [data['patient_count'][i] for i in sorted_indices]
+
+# Re-plot with sorted x-axis:
+ax.clear()
+bars = ax.bar(visit_sorted, count_sorted, color='darkblue', label='Patients by Visit Count')
+
+# Re-apply labels, etc.
+ax.set_xlabel('Number of Visits', fontsize=12)
+ax.set_ylabel('Number of Unique Patients', fontsize=12)
+#ax.set_title('Number of Patients by Visit Number', fontsize=14)
+
+# Add legend
+ax.legend()
+
+# Improve layout and grid
+ax.grid(axis='y', linestyle='--', alpha=0.7)
+plt.xticks(visit_sorted)  # Ensure all integer visit numbers are shown
+
+# Show the plot
+plt.tight_layout()
+plt.show()
+
+##

total_app.py

@@ -0,0 +1,149 @@
+import time
+import json
+import os
+from datetime import datetime
+import pandas as pd
+from openai import OpenAI
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# === CONFIGURATION ===
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL")
+MODEL_NAME = "GPT-OSS-120B"
+HEALTH_URL = f"{OPENAI_BASE_URL}/health"  # Placeholder - actual health check would need to be implemented
+CHAT_URL = f"{OPENAI_BASE_URL}/chat/completions"
+# File paths
+INPUT_CSV = "/home/shahin/Lab/Doktorarbeit/Barcelona/Data/MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_unique.csv"
+EDSS_INSTRUCTIONS_PATH = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/Komplett.txt"
+#GRAMMAR_FILE = "/home/shahin/Lab/Doktorarbeit/Barcelona/attach/just_edss_schema.gbnf"
+# Initialize OpenAI client
+client = OpenAI(
+    api_key=OPENAI_API_KEY,
+    base_url=OPENAI_BASE_URL
+)
+# Read EDSS instructions from file
+with open(EDSS_INSTRUCTIONS_PATH, 'r') as f:
+    EDSS_INSTRUCTIONS = f.read().strip()
+
+# === RUN INFERENCE 2 ===
+def run_inference(patient_text, max_retries=3):
+    prompt = f'''Du bist ein medizinischer Assistent, der spezialisiert darauf ist, EDSS-Scores (Expanded Disability Status Scale) sowie alle Unterkategorien aus klinischen Berichten zu extrahieren.
+### Regeln für die Ausgabe:
+1. **Reason**: Erstelle eine prägnante Zusammenfassung (max. 400 Zeichen) der Befunde auf **DEUTSCH**, die zur Einstufung führen.
+2. **klassifizierbar**:
+   - Setze dies auf **true**, wenn ein EDSS-Wert identifiziert, berechnet oder basierend auf den klinischen Hinweisen plausibel geschätzt werden kann.
+   - Setze dies auf **false**, NUR wenn die Daten absolut unzureichend oder so widersprüchlich sind, dass keinerlei Einstufung möglich ist.
+3. **EDSS**:
+   - Dieses Feld ist **VERPFLICHTEND**, wenn "klassifizierbar" auf true steht.
+   - Es muss eine Zahl zwischen 0.0 und 10.0 sein.
+   - Versuche stets, den EDSS-Wert so präzise wie möglich zu bestimmen, auch wenn die Datenlage dünn ist (nutze verfügbare Informationen zu Gehstrecke und Funktionssystemen).
+   - Dieses Feld **DARF NICHT ERSCHEINEN**, wenn "klassifizierbar" auf false steht.
+4. **Unterkategorien**:
+   - Extrahiere alle folgenden Unterkategorien aus dem Bericht:
+   - VISUAL OPTIC FUNCTIONS (max. 6.0)
+   - BRAINSTEM FUNCTIONS (max. 6.0)
+   - PYRAMIDAL FUNCTIONS (max. 6.0)
+   - CEREBELLAR FUNCTIONS (max. 6.0)
+   - SENSORY FUNCTIONS (max. 6.0)
+   - BOWEL AND BLADDER FUNCTIONS (max. 6.0)
+   - CEREBRAL FUNCTIONS (max. 6.0)
+   - AMBULATION (max. 10.0)
+   - Jede Unterkategorie sollte eine Zahl zwischen 0.0 und der jeweiligen Obergrenze enthalten, wenn sie klassifizierbar ist
+   - Wenn eine Unterkategorie nicht klassifizierbar ist, setze den Wert auf null
+### Einschränkungen:
+- Erfinde keine Fakten, aber nutze klinische Herleitungen aus dem Bericht, um den EDSS und die Unterkategorien zu bestimmen.
+- Priorisiere die Vergabe eines EDSS-Wertes gegenüber der Markierung als nicht klassifizierbar.
+- Halte dich strikt an die JSON-Struktur.
+- Die Unterkategorien müssen immer enthalten sein, auch wenn sie null sind.
+EDSS-Bewertungsrichtlinien:
+{EDSS_INSTRUCTIONS}
+Patientenbericht:
+{patient_text}
+'''
+
+    start_time = time.time()
+    for attempt in range(max_retries + 1):
+        try:
+            response = client.chat.completions.create(
+                messages=[
+                    {
+                        "role": "system",
+                        "content": "You extract EDSS scores and all subcategories. You prioritize providing values even if data is partial, by using clinical inference."
+                    },
+                    {
+                        "role": "user",
+                        "content": prompt
+                    }
+                ],
+                model=MODEL_NAME,
+                max_tokens=2048,
+                temperature=0.0,
+                response_format={"type": "json_object"}
+            )
+            content = response.choices[0].message.content
+
+            if content is None or content.strip() == "":
+                raise ValueError("API returned empty or None response content")
+
+            parsed = json.loads(content)
+            inference_time = time.time() - start_time
+            return {
+                "success": True,
+                "result": parsed,
+                "inference_time_sec": inference_time
+            }
+
+        except Exception as e:
+            print(f"Attempt {attempt + 1} failed: {e}")
+            if attempt < max_retries:
+                time.sleep(2 ** attempt)  # Exponential backoff
+                continue
+            else:
+                print("All retries exhausted.")
+                return {
+                    "success": False,
+                    "error": str(e),
+                    "inference_time_sec": -1
+                }
+
+# === BUILD PATIENT TEXT ===
+def build_patient_text(row):
+    # Handle potential NaN or None values in the row
+    summary = str(row.get("T_Zusammenfassung", "")) if pd.notna(row.get("T_Zusammenfassung")) else ""
+    diagnoses = str(row.get("Diagnosen", "")) if pd.notna(row.get("Diagnosen")) else ""
+    clinical = str(row.get("T_KlinBef", "")) if pd.notna(row.get("T_KlinBef")) else ""
+    findings = str(row.get("T_Befunde", "")) if pd.notna(row.get("T_Befunde")) else ""
+    return "\n".join([summary, diagnoses, clinical, findings]).strip()
+if __name__ == "__main__":
+    # Read CSV file ONLY inside main block
+    df = pd.read_csv(INPUT_CSV, sep=';')
+    results = []
+    # Process each row
+    for idx, row in df.iterrows():
+        print(f"Processing row {idx + 1}/{len(df)}")
+        try:
+            patient_text = build_patient_text(row)
+            result = run_inference(patient_text)
+            # Add unique_id and MedDatum to result for tracking
+            result["unique_id"] = row.get("unique_id", f"row_{idx}")
+            result["MedDatum"] = row.get("MedDatum", None)
+            results.append(result)
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+        except Exception as e:
+            print(f"Error processing row {idx}: {e}")
+            results.append({
+                "success": False,
+                "error": str(e),
+                "unique_id": row.get("unique_id", f"row_{idx}"),
+                "MedDatum": row.get("MedDatum", None)
+            })
+    # Save results to a JSON file
+    output_json = INPUT_CSV.replace(".csv", "_results_total.json")
+    with open(output_json, 'w', encoding='utf-8') as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+    print(f"Results saved to {output_json}")
+
+