632 lines
20 KiB
Python
632 lines
20 KiB
Python
# %% Scatter
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
|
||
# Load your data from TSV file
|
||
file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/join_MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_results+MS_Briefe_400_with_unique_id_SHA3_explore_cleaned.tsv'
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Replace comma with dot for numeric conversion in GT_EDSS and LLM_Results
|
||
df['GT_EDSS'] = df['GT_EDSS'].astype(str).str.replace(',', '.')
|
||
df['LLM_Results'] = df['LLM_Results'].astype(str).str.replace(',', '.')
|
||
|
||
# Convert to float (handle invalid entries gracefully)
|
||
df['GT_EDSS'] = pd.to_numeric(df['GT_EDSS'], errors='coerce')
|
||
df['LLM_Results'] = pd.to_numeric(df['LLM_Results'], errors='coerce')
|
||
|
||
# Drop rows where either column is NaN
|
||
df_clean = df.dropna(subset=['GT_EDSS', 'LLM_Results'])
|
||
|
||
# Create scatter plot
|
||
plt.figure(figsize=(8, 6))
|
||
plt.scatter(df_clean['GT_EDSS'], df_clean['LLM_Results'], alpha=0.7, color='blue')
|
||
|
||
# Add labels and title
|
||
plt.xlabel('GT_EDSS')
|
||
plt.ylabel('LLM_Results')
|
||
plt.title('Comparison of GT_EDSS vs LLM_Results')
|
||
|
||
# Optional: Add a diagonal line for reference (perfect prediction)
|
||
plt.plot([0, max(df_clean['GT_EDSS'])], [0, max(df_clean['GT_EDSS'])], color='red', linestyle='--', label='Perfect Prediction')
|
||
plt.legend()
|
||
|
||
# Show plot
|
||
plt.grid(True)
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
##
|
||
|
||
|
||
# %% Bland0-altman
|
||
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
import statsmodels.api as sm
|
||
|
||
# Load your data from TSV file
|
||
file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/join_MS_Briefe_400_with_unique_id_SHA3_explore_cleaned_results+MS_Briefe_400_with_unique_id_SHA3_explore_cleaned.tsv'
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Replace comma with dot for numeric conversion in GT_EDSS and LLM_Results
|
||
df['GT_EDSS'] = df['GT_EDSS'].astype(str).str.replace(',', '.')
|
||
df['LLM_Results'] = df['LLM_Results'].astype(str).str.replace(',', '.')
|
||
|
||
# Convert to float (handle invalid entries gracefully)
|
||
df['GT_EDSS'] = pd.to_numeric(df['GT_EDSS'], errors='coerce')
|
||
df['LLM_Results'] = pd.to_numeric(df['LLM_Results'], errors='coerce')
|
||
|
||
# Drop rows where either column is NaN
|
||
df_clean = df.dropna(subset=['GT_EDSS', 'LLM_Results'])
|
||
|
||
# Create Bland-Altman plot
|
||
f, ax = plt.subplots(1, figsize=(8, 5))
|
||
sm.graphics.mean_diff_plot(df_clean['GT_EDSS'], df_clean['LLM_Results'], ax=ax)
|
||
|
||
# Add labels and title
|
||
ax.set_title('Bland-Altman Plot: GT_EDSS vs LLM_Results')
|
||
ax.set_xlabel('Mean of GT_EDSS and LLM_Results')
|
||
ax.set_ylabel('Difference between GT_EDSS and LLM_Results')
|
||
|
||
# Display Bland-Altman plot
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
# Print some statistics
|
||
mean_diff = np.mean(df_clean['GT_EDSS'] - df_clean['LLM_Results'])
|
||
std_diff = np.std(df_clean['GT_EDSS'] - df_clean['LLM_Results'])
|
||
print(f"Mean difference: {mean_diff:.3f}")
|
||
print(f"Standard deviation of differences: {std_diff:.3f}")
|
||
print(f"95% Limits of Agreement: [{mean_diff - 1.96*std_diff:.3f}, {mean_diff + 1.96*std_diff:.3f}]")
|
||
|
||
##
|
||
|
||
|
||
|
||
# %% Confusion matrix
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.metrics import confusion_matrix, classification_report
|
||
import seaborn as sns
|
||
|
||
# Load your data from TSV file
|
||
file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv'
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Replace comma with dot for numeric conversion in GT.EDSS and result.EDSS
|
||
df['GT.EDSS'] = df['GT.EDSS'].astype(str).str.replace(',', '.')
|
||
df['result.EDSS'] = df['result.EDSS'].astype(str).str.replace(',', '.')
|
||
|
||
# Convert to float (handle invalid entries gracefully)
|
||
df['GT.EDSS'] = pd.to_numeric(df['GT.EDSS'], errors='coerce')
|
||
df['result.EDSS'] = pd.to_numeric(df['result.EDSS'], errors='coerce')
|
||
|
||
# Drop rows where either column is NaN
|
||
df_clean = df.dropna(subset=['GT.EDSS', 'result.EDSS'])
|
||
|
||
# For confusion matrix, we need to categorize the values
|
||
# Let's create categories up to 10 (0-1, 1-2, 2-3, ..., 9-10)
|
||
def categorize_edss(value):
|
||
if pd.isna(value):
|
||
return np.nan
|
||
elif value <= 1.0:
|
||
return '0-1'
|
||
elif value <= 2.0:
|
||
return '1-2'
|
||
elif value <= 3.0:
|
||
return '2-3'
|
||
elif value <= 4.0:
|
||
return '3-4'
|
||
elif value <= 5.0:
|
||
return '4-5'
|
||
elif value <= 6.0:
|
||
return '5-6'
|
||
elif value <= 7.0:
|
||
return '6-7'
|
||
elif value <= 8.0:
|
||
return '7-8'
|
||
elif value <= 9.0:
|
||
return '8-9'
|
||
elif value <= 10.0:
|
||
return '9-10'
|
||
else:
|
||
return '10+'
|
||
|
||
# Create categorical versions
|
||
df_clean['GT.EDSS_cat'] = df_clean['GT.EDSS'].apply(categorize_edss)
|
||
df_clean['result.EDSS_cat'] = df_clean['result.EDSS'].apply(categorize_edss)
|
||
|
||
# Remove any NaN categories
|
||
df_clean = df_clean.dropna(subset=['GT.EDSS_cat', 'result.EDSS_cat'])
|
||
|
||
# Create confusion matrix
|
||
cm = confusion_matrix(df_clean['GT.EDSS_cat'], df_clean['result.EDSS_cat'],
|
||
labels=['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'])
|
||
|
||
# Plot confusion matrix
|
||
plt.figure(figsize=(10, 8))
|
||
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
|
||
xticklabels=['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'],
|
||
yticklabels=['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10'])
|
||
plt.title('Confusion Matrix: Ground truth EDSS vs interferred EDSS (Categorized 0-10)')
|
||
plt.xlabel('LLM Generated EDSS')
|
||
plt.ylabel('Ground Truth EDSS')
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
# Print classification report
|
||
print("Classification Report:")
|
||
print(classification_report(df_clean['GT.EDSS_cat'], df_clean['result.EDSS_cat']))
|
||
|
||
# Print raw counts
|
||
print("\nConfusion Matrix (Raw Counts):")
|
||
print(cm)
|
||
|
||
##
|
||
|
||
|
||
|
||
# %% Classification
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
from sklearn.metrics import confusion_matrix
|
||
import numpy as np
|
||
|
||
# Load your data from TSV file
|
||
file_path ='/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv'
|
||
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Check data structure
|
||
print("Data shape:", df.shape)
|
||
print("First few rows:")
|
||
print(df.head())
|
||
print("\nColumn names:")
|
||
for col in df.columns:
|
||
print(f" {col}")
|
||
|
||
# Function to safely convert to boolean
|
||
def safe_bool_convert(series):
|
||
'''Safely convert series to boolean, handling various input formats'''
|
||
# Convert to string first, then to boolean
|
||
series_str = series.astype(str).str.strip().str.lower()
|
||
|
||
# Handle different true/false representations
|
||
bool_map = {
|
||
'true': True, '1': True, 'yes': True, 'y': True,
|
||
'false': False, '0': False, 'no': False, 'n': False
|
||
}
|
||
|
||
converted = series_str.map(bool_map)
|
||
|
||
# Handle remaining NaN values
|
||
converted = converted.fillna(False) # or True, depending on your preference
|
||
|
||
return converted
|
||
|
||
# Convert columns safely
|
||
if 'result.klassifizierbar' in df.columns:
|
||
print("\nresult.klassifizierbar column info:")
|
||
print(df['result.klassifizierbar'].head(10))
|
||
print("Unique values:", df['result.klassifizierbar'].unique())
|
||
|
||
df['result.klassifizierbar'] = safe_bool_convert(df['result.klassifizierbar'])
|
||
print("After conversion:")
|
||
print(df['result.klassifizierbar'].value_counts())
|
||
|
||
if 'GT.klassifizierbar' in df.columns:
|
||
print("\nGT.klassifizierbar column info:")
|
||
print(df['GT.klassifizierbar'].head(10))
|
||
print("Unique values:", df['GT.klassifizierbar'].unique())
|
||
|
||
df['GT.klassifizierbar'] = safe_bool_convert(df['GT.klassifizierbar'])
|
||
print("After conversion:")
|
||
print(df['GT.klassifizierbar'].value_counts())
|
||
|
||
# Create bar chart showing only True values for klassifizierbar
|
||
if 'result.klassifizierbar' in df.columns and 'GT.klassifizierbar' in df.columns:
|
||
# Get counts for True values only
|
||
llm_true_count = df['result.klassifizierbar'].sum()
|
||
gt_true_count = df['GT.klassifizierbar'].sum()
|
||
|
||
# Plot using matplotlib directly
|
||
fig, ax = plt.subplots(figsize=(8, 6))
|
||
|
||
x = np.arange(2)
|
||
width = 0.35
|
||
|
||
bars1 = ax.bar(x[0] - width/2, llm_true_count, width, label='LLM', color='skyblue', alpha=0.8)
|
||
bars2 = ax.bar(x[1] + width/2, gt_true_count, width, label='GT', color='lightcoral', alpha=0.8)
|
||
|
||
# Add value labels on bars
|
||
ax.annotate(f'{llm_true_count}',
|
||
xy=(x[0], llm_true_count),
|
||
xytext=(0, 3),
|
||
textcoords="offset points",
|
||
ha='center', va='bottom')
|
||
|
||
ax.annotate(f'{gt_true_count}',
|
||
xy=(x[1], gt_true_count),
|
||
xytext=(0, 3),
|
||
textcoords="offset points",
|
||
ha='center', va='bottom')
|
||
|
||
ax.set_xlabel('Classification Status (klassifizierbar)')
|
||
ax.set_ylabel('Count')
|
||
ax.set_title('True Values Comparison: LLM vs GT for "klassifizierbar"')
|
||
ax.set_xticks(x)
|
||
ax.set_xticklabels(['LLM', 'GT'])
|
||
ax.legend()
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
# Create confusion matrix if both columns exist
|
||
if 'result.klassifizierbar' in df.columns and 'GT.klassifizierbar' in df.columns:
|
||
try:
|
||
# Ensure both columns are boolean
|
||
llm_bool = df['result.klassifizierbar'].fillna(False).astype(bool)
|
||
gt_bool = df['GT.klassifizierbar'].fillna(False).astype(bool)
|
||
|
||
cm = confusion_matrix(gt_bool, llm_bool)
|
||
|
||
# Plot confusion matrix
|
||
fig, ax = plt.subplots(figsize=(8, 6))
|
||
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
|
||
xticklabels=['False ', 'True '],
|
||
yticklabels=['False', 'True '],
|
||
ax=ax)
|
||
ax.set_xlabel('LLM Predictions ')
|
||
ax.set_ylabel('GT Labels ')
|
||
ax.set_title('Confusion Matrix: LLM vs GT for "klassifizierbar"')
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
print("Confusion Matrix:")
|
||
print(cm)
|
||
|
||
except Exception as e:
|
||
print(f"Error creating confusion matrix: {e}")
|
||
|
||
# Show final data info
|
||
print("\nFinal DataFrame info:")
|
||
print(df[['result.klassifizierbar', 'GT.klassifizierbar']].info())
|
||
|
||
##
|
||
|
||
|
||
|
||
|
||
# %% Boxplot
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
import numpy as np
|
||
|
||
# Load your data from TSV file
|
||
file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/join_results_unique.tsv'
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Replace comma with dot for numeric conversion in GT.EDSS and result.EDSS
|
||
df['GT.EDSS'] = df['GT.EDSS'].astype(str).str.replace(',', '.')
|
||
df['result.EDSS'] = df['result.EDSS'].astype(str).str.replace(',', '.')
|
||
|
||
# Convert to float (handle invalid entries gracefully)
|
||
df['GT.EDSS'] = pd.to_numeric(df['GT.EDSS'], errors='coerce')
|
||
df['result.EDSS'] = pd.to_numeric(df['result.EDSS'], errors='coerce')
|
||
|
||
# Drop rows where either column is NaN
|
||
df_clean = df.dropna(subset=['GT.EDSS', 'result.EDSS'])
|
||
|
||
# 1. DEFINE CATEGORY ORDER
|
||
# This ensures the X-axis is numerically logical (0-1 comes before 1-2)
|
||
category_order = ['0-1', '1-2', '2-3', '3-4', '4-5', '5-6', '6-7', '7-8', '8-9', '9-10', '10+']
|
||
|
||
# Convert the column to a Categorical type with the specific order
|
||
df_clean['GT.EDSS_cat'] = pd.Categorical(df_clean['GT.EDSS'].apply(categorize_edss),
|
||
categories=category_order,
|
||
ordered=True)
|
||
|
||
plt.figure(figsize=(14, 8))
|
||
|
||
# 2. ADD HUE FOR LEGEND
|
||
# Assigning x to 'hue' allows Seaborn to generate a legend automatically
|
||
box_plot = sns.boxplot(
|
||
data=df_clean,
|
||
x='GT.EDSS_cat',
|
||
y='result.EDSS',
|
||
hue='GT.EDSS_cat', # Added hue
|
||
palette='viridis',
|
||
linewidth=1.5,
|
||
legend=True # Ensure legend is enabled
|
||
)
|
||
|
||
# 3. CUSTOMIZE PLOT
|
||
plt.title('Distribution of result.EDSS by GT.EDSS Category', fontsize=18, pad=20)
|
||
plt.xlabel('Ground Truth EDSS Category', fontsize=14)
|
||
plt.ylabel('LLM Predicted EDSS', fontsize=14)
|
||
|
||
# Move legend to the side or top
|
||
plt.legend(title="EDSS Categories", bbox_to_anchor=(1.05, 1), loc='upper left')
|
||
|
||
plt.xticks(rotation=45, ha='right', fontsize=10)
|
||
plt.grid(True, axis='y', alpha=0.3)
|
||
plt.tight_layout()
|
||
|
||
plt.show()
|
||
##
|
||
|
||
|
||
# %% Postproccessing Column names
|
||
|
||
import pandas as pd
|
||
|
||
# Read the TSV file
|
||
file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv'
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Create a mapping dictionary for German to English column names
|
||
column_mapping = {
|
||
'EDSS':'GT.EDSS',
|
||
'klassifizierbar': 'GT.klassifizierbar',
|
||
'Sehvermögen': 'GT.VISUAL_OPTIC_FUNCTIONS',
|
||
'Cerebellum': 'GT.CEREBELLAR_FUNCTIONS',
|
||
'Hirnstamm': 'GT.BRAINSTEM_FUNCTIONS',
|
||
'Sensibiliät': 'GT.SENSORY_FUNCTIONS',
|
||
'Pyramidalmotorik': 'GT.PYRAMIDAL_FUNCTIONS',
|
||
'Ambulation': 'GT.AMBULATION',
|
||
'Cerebrale_Funktion': 'GT.CEREBRAL_FUNCTIONS',
|
||
'Blasen-_und_Mastdarmfunktion': 'GT.BOWEL_AND_BLADDER_FUNCTIONS'
|
||
}
|
||
|
||
# Rename columns
|
||
df = df.rename(columns=column_mapping)
|
||
|
||
# Save the modified dataframe back to TSV file
|
||
df.to_csv(file_path, sep='\t', index=False)
|
||
|
||
print("Columns have been successfully renamed!")
|
||
print("Renamed columns:")
|
||
for old_name, new_name in column_mapping.items():
|
||
if old_name in df.columns:
|
||
print(f" {old_name} -> {new_name}")
|
||
|
||
|
||
##
|
||
|
||
|
||
|
||
|
||
# %% Styled table
|
||
import pandas as pd
|
||
import numpy as np
|
||
import seaborn as sns
|
||
import matplotlib.pyplot as plt
|
||
import dataframe_image as dfi
|
||
# Load data
|
||
df = pd.read_csv("/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv", sep='\t')
|
||
|
||
# 1. Identify all GT and result columns
|
||
gt_columns = [col for col in df.columns if col.startswith('GT.')]
|
||
result_columns = [col for col in df.columns if col.startswith('result.')]
|
||
|
||
print("GT Columns found:", gt_columns)
|
||
print("Result Columns found:", result_columns)
|
||
|
||
# 2. Create proper mapping between GT and result columns
|
||
# Handle various naming conventions (spaces, underscores, etc.)
|
||
column_mapping = {}
|
||
|
||
for gt_col in gt_columns:
|
||
base_name = gt_col.replace('GT.', '')
|
||
|
||
# Clean the base name for matching - remove spaces, underscores, etc.
|
||
# Try different matching approaches
|
||
candidates = [
|
||
f'result.{base_name}', # Exact match
|
||
f'result.{base_name.replace(" ", "_")}', # With underscores
|
||
f'result.{base_name.replace("_", " ")}', # With spaces
|
||
f'result.{base_name.replace(" ", "")}', # No spaces
|
||
f'result.{base_name.replace("_", "")}' # No underscores
|
||
]
|
||
|
||
# Also try case-insensitive matching
|
||
candidates.append(f'result.{base_name.lower()}')
|
||
candidates.append(f'result.{base_name.upper()}')
|
||
|
||
# Try to find matching result column
|
||
matched = False
|
||
for candidate in candidates:
|
||
if candidate in result_columns:
|
||
column_mapping[gt_col] = candidate
|
||
matched = True
|
||
break
|
||
|
||
# If no exact match found, try partial matching
|
||
if not matched:
|
||
# Try to match by removing special characters and comparing
|
||
base_clean = ''.join(e for e in base_name if e.isalnum() or e in ['_', ' '])
|
||
for result_col in result_columns:
|
||
result_base = result_col.replace('result.', '')
|
||
result_clean = ''.join(e for e in result_base if e.isalnum() or e in ['_', ' '])
|
||
if base_clean.lower() == result_clean.lower():
|
||
column_mapping[gt_col] = result_col
|
||
matched = True
|
||
break
|
||
|
||
print("Column mapping:", column_mapping)
|
||
|
||
# 3. Faster, vectorized computation using the corrected mapping
|
||
data_list = []
|
||
|
||
for gt_col, result_col in column_mapping.items():
|
||
print(f"Processing {gt_col} vs {result_col}")
|
||
|
||
# Convert to numeric, forcing errors to NaN
|
||
s1 = pd.to_numeric(df[gt_col], errors='coerce').astype(float)
|
||
s2 = pd.to_numeric(df[result_col], errors='coerce').astype(float)
|
||
|
||
# Calculate matches (abs difference <= 0.5)
|
||
diff = np.abs(s1 - s2)
|
||
matches = (diff <= 0.5).sum()
|
||
|
||
# Determine the denominator (total valid comparisons)
|
||
valid_count = diff.notna().sum()
|
||
|
||
if valid_count > 0:
|
||
percentage = (matches / valid_count) * 100
|
||
else:
|
||
percentage = 0
|
||
|
||
# Extract clean base name for display
|
||
base_name = gt_col.replace('GT.', '')
|
||
|
||
data_list.append({
|
||
'GT': base_name,
|
||
'Match %': round(percentage, 1)
|
||
})
|
||
|
||
|
||
|
||
|
||
# 4. Prepare Data
|
||
match_df = pd.DataFrame(data_list)
|
||
# Clean up labels: Replace underscores with spaces and capitalize
|
||
match_df['GT'] = match_df['GT'].str.replace('_', ' ').str.title()
|
||
match_df = match_df.sort_values('Match %', ascending=False)
|
||
|
||
# 5. Create a "Beautiful" Table using Seaborn Heatmap
|
||
def create_luxury_table(df, output_file="edss_agreement.png"):
|
||
# Set the aesthetic style
|
||
sns.set_theme(style="white", font="sans-serif")
|
||
|
||
# Prepare data for heatmap
|
||
plot_data = df.set_index('GT')[['Match %']]
|
||
|
||
# Initialize the figure
|
||
# Height is dynamic based on number of rows
|
||
fig, ax = plt.subplots(figsize=(8, len(df) * 0.6))
|
||
|
||
# Create a custom diverging color map (Deep Red -> Mustard -> Emerald)
|
||
# This looks more professional than standard 'RdYlGn'
|
||
cmap = sns.diverging_palette(15, 135, s=80, l=55, as_cmap=True)
|
||
|
||
# Draw the heatmap
|
||
sns.heatmap(
|
||
plot_data,
|
||
annot=True,
|
||
fmt=".1f",
|
||
cmap=cmap,
|
||
center=85, # Centers the color transition
|
||
vmin=50, vmax=100, # Range of the gradient
|
||
linewidths=2,
|
||
linecolor='white',
|
||
cbar=False, # Remove color bar for a "table" look
|
||
annot_kws={"size": 14, "weight": "bold", "family": "sans-serif"}
|
||
)
|
||
|
||
# Styling the Axes (Turning the heatmap into a table)
|
||
ax.set_xlabel("")
|
||
ax.set_ylabel("")
|
||
ax.xaxis.tick_top() # Move "Match %" label to top
|
||
ax.set_xticklabels(['Agreement (%)'], fontsize=14, fontweight='bold', color='#2c3e50')
|
||
ax.tick_params(axis='y', labelsize=12, labelcolor='#2c3e50', length=0)
|
||
|
||
# Add a thin border around the plot
|
||
for _, spine in ax.spines.items():
|
||
spine.set_visible(True)
|
||
spine.set_color('#ecf0f1')
|
||
|
||
plt.title('EDSS Subcategory Consistency Analysis', fontsize=16, pad=40, fontweight='bold', color='#2c3e50')
|
||
|
||
# Add a subtle footer
|
||
plt.figtext(0.5, 0.0, "Tolerance: ±0.5 points",
|
||
wrap=True, horizontalalignment='center', fontsize=10, color='gray', style='italic')
|
||
|
||
# Save with high resolution
|
||
plt.tight_layout()
|
||
plt.savefig(output_file, dpi=300, bbox_inches='tight')
|
||
print(f"Beautiful table saved as {output_file}")
|
||
|
||
# Execute
|
||
create_luxury_table(match_df)
|
||
|
||
|
||
# Run the function
|
||
save_styled_table(match_df)
|
||
# 6. Save as SVG
|
||
|
||
plt.savefig("agreement_table.svg", format='svg', dpi=300, bbox_inches='tight')
|
||
print("Successfully saved agreement_table.svg")
|
||
|
||
# Show plot if running in a GUI environment
|
||
plt.show()
|
||
##
|
||
|
||
|
||
|
||
# %% Time Plot
|
||
import numpy as np
|
||
import matplotlib.pyplot as plt
|
||
import pandas as pd
|
||
from scipy import stats
|
||
|
||
# Load the TSV file
|
||
file_path = '/home/shahin/Lab/Doktorarbeit/Barcelona/Data/Join_edssandsub.tsv'
|
||
df = pd.read_csv(file_path, sep='\t')
|
||
|
||
# Extract the inference_time_sec column
|
||
inference_times = df['inference_time_sec'].dropna() # Remove NaN values
|
||
|
||
# Calculate statistics
|
||
mean_time = inference_times.mean()
|
||
std_time = inference_times.std()
|
||
median_time = np.median(inference_times)
|
||
|
||
# Create the histogram
|
||
fig, ax = plt.subplots(figsize=(10, 6))
|
||
|
||
# Create histogram with bins of 1 second width
|
||
min_time = int(inference_times.min())
|
||
max_time = int(inference_times.max()) + 1
|
||
bins = np.arange(min_time, max_time + 1, 1) # Bins of 1 second width
|
||
|
||
# Create histogram with counts (not probability density)
|
||
n, bins, patches = ax.hist(inference_times, bins=bins, color='lightblue', alpha=0.7, edgecolor='black', linewidth=0.5)
|
||
|
||
# Generate Gaussian curve for fit
|
||
x = np.linspace(inference_times.min(), inference_times.max(), 100)
|
||
# Scale Gaussian to match histogram counts
|
||
gaussian_counts = stats.norm.pdf(x, mean_time, std_time) * len(inference_times) * (bins[1] - bins[0])
|
||
|
||
# Plot Gaussian fit
|
||
ax.plot(x, gaussian_counts, color='red', linewidth=2, label=f'Gaussian Fit (μ={mean_time:.1f}s, σ={std_time:.1f}s)')
|
||
|
||
# Add vertical lines for mean and median
|
||
ax.axvline(mean_time, color='blue', linestyle='--', linewidth=2, label=f'Mean = {mean_time:.1f}s')
|
||
ax.axvline(median_time, color='green', linestyle='--', linewidth=2, label=f'Median = {median_time:.1f}s')
|
||
|
||
# Add standard deviation as vertical lines
|
||
ax.axvline(mean_time + std_time, color='saddlebrown', linestyle=':', linewidth=1, alpha=0.7, label=f'+1σ = {mean_time + std_time:.1f}s')
|
||
ax.axvline(mean_time - std_time, color='saddlebrown', linestyle=':', linewidth=1, alpha=0.7, label=f'-1σ = {mean_time - std_time:.1f}s')
|
||
|
||
ax.set_xlabel('Inference Time (seconds)')
|
||
ax.set_ylabel('Frequency')
|
||
ax.set_title('Inference Time Distribution with Gaussian Fit')
|
||
ax.legend()
|
||
ax.grid(True, alpha=0.3)
|
||
|
||
plt.tight_layout()
|
||
plt.show()
|
||
|
||
##
|
||
|
||
|
||
|
||
|