{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1299f52c",
   "metadata": {},
   "source": [
    "# Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cb135086",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import os\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "sns.set(style='whitegrid')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef079d1a",
   "metadata": {},
   "source": [
    "## Training set size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd01c684",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_accuracies = []\n",
    "all_precisions = []\n",
    "all_recalls = []\n",
    "all_f1_scores = []\n",
    "all_memory_usages = []\n",
    "all_model_sizes = []\n",
    "\n",
    "all_delta_accuracies = []\n",
    "all_delta_precisions = []\n",
    "all_delta_recalls = []\n",
    "all_delta_f1_scores = []\n",
    "all_delta_memory_usages = []\n",
    "all_delta_model_sizes = []\n",
    "\n",
    "for seed in range(0, 50, 10):\n",
    "    df = pd.read_csv(f'./data/results/training_size/training_size_results_seed{seed}.csv')\n",
    "    \n",
    "    all_accuracies.append(df['accuracy'].to_list())\n",
    "    all_precisions.append(df['precision'].to_list())\n",
    "    all_recalls.append(df['recall'].to_list())\n",
    "    all_f1_scores.append(df['f1-score'].to_list())\n",
    "    all_memory_usages.append(df['memory_usage'].to_list())\n",
    "    all_model_sizes.append(df['model_size'].to_list())\n",
    "    \n",
    "    all_delta_accuracies.append(df['accuracy'].diff().values.tolist())\n",
    "    all_delta_precisions.append(df['precision'].diff().values.tolist())\n",
    "    all_delta_recalls.append(df['recall'].diff().values.tolist())\n",
    "    all_delta_f1_scores.append(df['f1-score'].diff().values.tolist())\n",
    "    all_delta_memory_usages.append(df['memory_usage'].diff().values.tolist())\n",
    "    all_delta_model_sizes.append(df['model_size'].diff().values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "adce9c6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def repeat_elements(ls, n):\n",
    "    return [element for element in ls for _ in range(n)]\n",
    "\n",
    "labels = repeat_elements(df['model'].to_list(), 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ebbdd9c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "def flatten(ls):\n",
    "    return list(itertools.chain(*list(zip(*ls))))\n",
    "\n",
    "accs = flatten(all_accuracies)\n",
    "prcs = flatten(all_precisions)\n",
    "recs = flatten(all_recalls)\n",
    "f1ss = flatten(all_f1_scores)\n",
    "mems = flatten(all_memory_usages)\n",
    "mods = flatten(all_model_sizes)\n",
    "\n",
    "delta_accs = flatten(all_delta_accuracies)\n",
    "delta_prcs = flatten(all_delta_precisions)\n",
    "delta_recs = flatten(all_delta_recalls)\n",
    "delta_f1ss = flatten(all_delta_f1_scores)\n",
    "delta_mems = flatten(all_delta_memory_usages)\n",
    "delta_mods = flatten(all_delta_model_sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "475beca6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 119052 characters]",
      "text/plain": [
       "<Figure size 800x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://seaborn.pydata.org/tutorial/error_bars.html\n",
    "\n",
    "left_color = 'cornflowerblue'\n",
    "right_color = 'olivedrab'\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "\n",
    "sns.pointplot(y=accs, x=labels, label='Accuracy', marker='o', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles='--', linewidth=1.2, markersize=5, color=left_color, alpha=0.8)\n",
    "sns.pointplot(y=prcs, x=labels, label='Precision', marker='D', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles='-.', linewidth=1.2, markersize=5, color=left_color, alpha=0.8)\n",
    "sns.pointplot(y=recs, x=labels, label='Recall', marker='s', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles=':', linewidth=1.2, markersize=5, color=left_color, alpha=0.8)\n",
    "ax = sns.pointplot(y=f1ss, x=labels, label='F1-Score', marker='X', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles='-', linewidth=1.2, markersize=6, color=left_color, alpha=0.8)\n",
    "ax.tick_params(axis='y', colors=left_color)\n",
    "ax.yaxis.grid(True)\n",
    "\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Performance', color=left_color, fontsize=16)\n",
    "plt.xticks(rotation=90, color='black', fontsize=12)\n",
    "plt.ylim(top=1)\n",
    "# Get the legend object\n",
    "legend = plt.legend()\n",
    "handles = legend.legend_handles\n",
    "for h in handles:\n",
    "    h.set_color('black')\n",
    "sns.move_legend(plt.gca(), loc='upper left', bbox_to_anchor=(0.75, 0.55), fontsize=12)\n",
    "\n",
    "ax2 = plt.twinx()\n",
    "\n",
    "sns.pointplot(y=delta_accs, x=labels, marker='o', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles='--', linewidth=1.2, markersize=5, color=right_color, alpha=0.8, ax=ax2)\n",
    "sns.pointplot(y=delta_prcs, x=labels, marker='D', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles='-.', linewidth=1.2, markersize=5, color=right_color, alpha=0.8, ax=ax2)\n",
    "sns.pointplot(y=delta_recs, x=labels, marker='s', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles=':', linewidth=1.2, markersize=5, color=right_color, alpha=0.8, ax=ax2)\n",
    "ax = sns.pointplot(y=delta_f1ss, x=labels, marker='X', errorbar='sd', err_kws={\"linewidth\": 2}, linestyles='-', linewidth=1.2, markersize=6, color=right_color, alpha=0.8, ax=ax2)\n",
    "\n",
    "ax.yaxis.grid(False)\n",
    "ax.tick_params(axis='y', colors=right_color)\n",
    "ax.spines['left'].set_color(left_color)\n",
    "ax.spines['right'].set_color(right_color)\n",
    "# ax.spines['bottom'].set_color('darkblue')\n",
    "ax.spines['left'].set_linewidth(1.1)\n",
    "ax.spines['right'].set_linewidth(1.1)\n",
    "ax.spines['bottom'].set_linewidth(1.1)\n",
    "ax.tick_params(width=1.1)\n",
    "\n",
    "sns.despine(top=True, right=False, left=False, bottom=False)\n",
    "plt.ylabel('Delta', color=right_color, fontsize=16)\n",
    "# plt.title(f'Learning curves by training size proportion', fontsize=18)\n",
    "plt.savefig('training-size_metrics.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7eb79a2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 79684 characters]",
      "text/plain": [
       "<Figure size 800x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "left_color = 'cornflowerblue'\n",
    "right_color = 'olivedrab'\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "\n",
    "ax = sns.pointplot(y=mods, x=labels, marker='o', errorbar='sd', capsize=.4, err_kws={\"linewidth\": 2}, linewidth=2, markersize=4, color=left_color)\n",
    "# ax = sns.lineplot(df_results, y='model_size', x='model', linewidth=2, marker='o', color=left_color)\n",
    "ax.tick_params(axis='y', colors=left_color)\n",
    "ax.yaxis.grid(True)\n",
    "\n",
    "plt.xticks(rotation=90, color='black', fontsize=12)\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Model file size (megabytes)', color=left_color, fontsize=16)\n",
    "\n",
    "ax2 = plt.twinx()\n",
    "\n",
    "ax = sns.pointplot(y=mems, x=labels, marker='o', errorbar='sd', capsize=.4, err_kws={\"linewidth\": 2}, linewidth=2, markersize=4, color=right_color, ax=ax2)\n",
    "# ax = sns.lineplot(df_results, y='memory_usage', x='model', linewidth=2, marker='o', color=right_color, ax=ax2)\n",
    "ax.yaxis.grid(False)\n",
    "ax.tick_params(axis='y', colors=right_color)\n",
    "ax.spines['left'].set_color(left_color)\n",
    "ax.spines['right'].set_color(right_color)\n",
    "# ax.spines['bottom'].set_color('darkblue')\n",
    "ax.spines['left'].set_linewidth(1.1)\n",
    "ax.spines['right'].set_linewidth(1.1)\n",
    "ax.spines['bottom'].set_linewidth(1.1)\n",
    "ax.tick_params(width=1.1)\n",
    "\n",
    "sns.despine(top=True, right=False, left=False, bottom=False)\n",
    "plt.ylabel('Memory usage (megabytes)', color=right_color, fontsize=16)\n",
    "# plt.title(f'Compute usage by training size proportion', fontsize=18)\n",
    "plt.savefig('training-size_memory.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1f9b37b",
   "metadata": {},
   "source": [
    "## Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f98f29d",
   "metadata": {},
   "source": [
    "### Training file size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1193d602",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_size(file_path):\n",
    "    return (os.path.getsize(file_path) / 1024) / 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00636007",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_file_size_df = []\n",
    "for seed in range(0, 50, 10):\n",
    "    df = pd.DataFrame({\n",
    "        'model': ['Baseline', 'Mutual Information', 'Markov Blanket', 'Boruta', 'RFE', 'Random Features'],\n",
    "        'train_file_size': [\n",
    "            get_file_size(f'./data/datasets/features/train_263-features_seed{seed}.csv'),\n",
    "            get_file_size(f'./data/datasets/features/train_features_mutual-information_seed{seed}.csv'),\n",
    "            get_file_size(f'./data/datasets/features/train_features_markov-blanket_seed{seed}.csv'),\n",
    "            get_file_size(f'./data/datasets/features/train_features_boruta_seed{seed}.csv'),\n",
    "            get_file_size(f'./data/datasets/features/train_features_rfe_seed{seed}.csv'),\n",
    "            get_file_size(f'./data/datasets/features/train_features_random-features_seed{seed}.csv')\n",
    "        ]\n",
    "    })\n",
    "    all_file_size_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "48926814",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 26168 characters]",
      "text/plain": [
       "<Figure size 600x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 3))\n",
    "\n",
    "sns.barplot(\n",
    "    pd.concat(all_file_size_df), \n",
    "    y='model', x='train_file_size', \n",
    "    errorbar=\"sd\", capsize=.4, \n",
    "    err_kws={\"color\": \".33\", \"linewidth\": 1.5}, \n",
    "    palette=[\"b\", \"orange\", \"darkgray\", \"r\", \"tab:purple\", \"yellow\"]\n",
    ")\n",
    "sns.despine(top=True, right=True, left=False, bottom=False)\n",
    "\n",
    "# plt.title('Training file size of features', fontsize=18)\n",
    "plt.ylabel('')\n",
    "plt.xlabel('File size (megabytes)')\n",
    "plt.savefig('training_file_sizes.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "864669a9",
   "metadata": {},
   "source": [
    "### Reductions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "069cb52d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_file_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.903010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.893359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.911765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.895267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.894104</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_file_size\n",
       "0         0.903010\n",
       "1         0.893359\n",
       "2         0.911765\n",
       "3         0.895267\n",
       "4         0.894104"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat(all_file_size_df).groupby(['model', 'train_file_size']).sum()\n",
    "(df.loc['Baseline'].reset_index() - df.loc['Markov Blanket'].reset_index()) / df.loc['Baseline'].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cb4a2c1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_file_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.897304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.887904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.904312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.889256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.887952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_file_size\n",
       "0         0.897304\n",
       "1         0.887904\n",
       "2         0.904312\n",
       "3         0.889256\n",
       "4         0.887952"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.loc['Baseline'].reset_index() - df.loc['RFE'].reset_index()) / df.loc['Baseline'].reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adec6a16",
   "metadata": {},
   "source": [
    "### Selected features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6d69b376",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\n",
    "    'model': ['Baseline', 'Mutual Information', 'Markov Blanket', 'Boruta', 'RFE', 'Random Features']*5,\n",
    "    'number_features': [\n",
    "        263, 66, 17, 105, 17, 17,\n",
    "        263, 66, 20, 109, 20, 20,\n",
    "        263, 66, 19, 107, 19, 19,\n",
    "        263, 66, 19, 108, 19, 19,\n",
    "        263, 66, 19, 108, 19, 19\n",
    "    ]\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1398cb6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47288 characters]",
      "text/plain": [
       "<Figure size 500x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 4))\n",
    "\n",
    "sns.barplot(\n",
    "    df, \n",
    "    x='model', y='number_features', \n",
    "    errorbar=\"sd\", capsize=.4, \n",
    "    err_kws={\"color\": \".33\", \"linewidth\": 1.5}, \n",
    "    palette=[\"b\", \"orange\", \"darkgray\", \"r\", \"tab:purple\", \"yellow\"]\n",
    ")\n",
    "sns.despine(top=True, right=True, left=False, bottom=False)\n",
    "\n",
    "plt.title('Number of selected features', fontsize=18)\n",
    "plt.ylabel('Number of features')\n",
    "plt.xlabel('')\n",
    "plt.xticks(rotation=30)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15755028",
   "metadata": {},
   "source": [
    "### Performance metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8cde5ad3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>variable</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Baseline</td>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.981469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mutual Information</td>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.979542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Markov Blanket</td>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.977601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Boruta</td>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.979940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RFE</td>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.976279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>Random Features</td>\n",
       "      <td>F1-Score</td>\n",
       "      <td>0.791385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>Random Features</td>\n",
       "      <td>F1-Score</td>\n",
       "      <td>0.563196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>Random Features</td>\n",
       "      <td>F1-Score</td>\n",
       "      <td>0.653227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>Random Features</td>\n",
       "      <td>F1-Score</td>\n",
       "      <td>0.569868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>699</th>\n",
       "      <td>Random Features</td>\n",
       "      <td>F1-Score</td>\n",
       "      <td>0.581146</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>700 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  model  variable     value\n",
       "0              Baseline  Accuracy  0.981469\n",
       "1    Mutual Information  Accuracy  0.979542\n",
       "2        Markov Blanket  Accuracy  0.977601\n",
       "3                Boruta  Accuracy  0.979940\n",
       "4                   RFE  Accuracy  0.976279\n",
       "..                  ...       ...       ...\n",
       "695     Random Features  F1-Score  0.791385\n",
       "696     Random Features  F1-Score  0.563196\n",
       "697     Random Features  F1-Score  0.653227\n",
       "698     Random Features  F1-Score  0.569868\n",
       "699     Random Features  F1-Score  0.581146\n",
       "\n",
       "[700 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_metrics_df = []\n",
    "for seed in range(0, 50, 10):\n",
    "    # baseline\n",
    "#     df_baseline = pd.read_csv(f'./data/results/training_size/training_size_results_seed{seed}.csv')\n",
    "#     df_baseline = df_baseline[['model', 'accuracy', 'precision', 'recall', 'f1-score']]\n",
    "#     df_baseline = df_baseline[df_baseline['model'] == '100.0%'].reset_index(drop=True)\n",
    "#     df_baseline = df_baseline.rename(columns={\n",
    "#         'accuracy': 'Accuracy',\n",
    "#         'precision': 'Precision',\n",
    "#         'recall': 'Recall',\n",
    "#         'f1-score': 'F1-Score',\n",
    "#     })\n",
    "#     df_baseline = df_baseline.replace({'100.0%': 'Baseline'})\n",
    "#     all_metrics_df.append(df_baseline)\n",
    "    \n",
    "    # feature selection\n",
    "    df = pd.read_csv(f'./data/results/feature_selection/metrics/feature_selection_metrics_test_seed{seed}.csv')\n",
    "    df = df.rename(columns={\n",
    "        'accuracy': 'Accuracy',\n",
    "        'precision': 'Precision',\n",
    "        'recall': 'Recall',\n",
    "        'f1-score': 'F1-Score',\n",
    "    })\n",
    "    df = df.replace(to_replace=r'RFE(.*)', value='RFE', regex=True)\n",
    "    all_metrics_df.append(df)\n",
    "    \n",
    "    # random features\n",
    "    df_random = pd.read_csv(f'./data/results/feature_selection/metrics/random_features_metrics_test_seed{seed}.csv')\n",
    "    df_random = df_random.rename(columns={\n",
    "        'accuracy': 'Accuracy',\n",
    "        'precision': 'Precision',\n",
    "        'recall': 'Recall',\n",
    "        'f1-score': 'F1-Score',\n",
    "    })\n",
    "    df_random = df_random.replace(to_replace=r'Random(.*)', value='Random Features', regex=True)\n",
    "    all_metrics_df.append(df_random)\n",
    "    \n",
    "df = pd.melt(pd.concat(all_metrics_df), id_vars=['model'], value_vars=['Accuracy', 'Precision', 'Recall', 'F1-Score'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d4565922",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37952 characters]",
      "text/plain": [
       "<Figure size 800x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 4))\n",
    "\n",
    "ax = sns.barplot(\n",
    "    df, x='variable', y='value', \n",
    "    hue='model', \n",
    "    errorbar=\"sd\", capsize=.4, \n",
    "    err_kws={\"color\": \".33\", \"linewidth\": 1.5},\n",
    "    palette=[\"b\", \"orange\", \"darkgray\", \"r\", \"tab:purple\", \"yellow\"]\n",
    ")\n",
    "sns.despine(top=True, right=True, left=False, bottom=False)\n",
    "sns.move_legend(ax, \"lower center\", bbox_to_anchor=(.5, 0.98), handletextpad=0.2, columnspacing=0.75, ncol=6, title=None, frameon=False)\n",
    "\n",
    "plt.title('Performance metrics on test set', y=1.1, fontsize=18)\n",
    "plt.setp(ax.get_legend().get_texts(), fontsize='11')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('')\n",
    "plt.ylim((0.5, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfd2e6c4",
   "metadata": {},
   "source": [
    "### Inference Time with Raspberry Pi 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ae85679d",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_time_df = []\n",
    "for seed in range(0, 50, 10):\n",
    "    df_mi = pd.read_csv(f'./data/results/inference_time/test_inference_time_mi_seed{seed}.csv')\n",
    "    df_mi = df_mi.rename(columns={'inference_time': 'Mutual Information'})\n",
    "\n",
    "    df_mb = pd.read_csv(f'./data/results/inference_time/test_inference_time_mb_seed{seed}.csv')\n",
    "    df_mb = df_mb.rename(columns={'inference_time': 'Markov Blanket'})\n",
    "\n",
    "    df_boruta = pd.read_csv(f'./data/results/inference_time/test_inference_time_boruta_seed{seed}.csv')\n",
    "    df_boruta = df_boruta.rename(columns={'inference_time': 'Boruta'})\n",
    "\n",
    "    df_rfe = pd.read_csv(f'./data/results/inference_time/test_inference_time_rfe_seed{seed}.csv')\n",
    "    df_rfe = df_rfe.rename(columns={'inference_time': 'RFE'})\n",
    "\n",
    "    df_random = pd.read_csv(f'./data/results/inference_time/test_inference_time_random-features_seed{seed}.csv')\n",
    "    df_random = df_random.rename(columns={'inference_time': 'Random Features'})\n",
    "    \n",
    "    df = pd.concat([\n",
    "        df_mi,\n",
    "        df_mb,\n",
    "        df_boruta,\n",
    "        df_rfe,\n",
    "        df_random\n",
    "    ], axis=1)\n",
    "    all_time_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9c82cca5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 46660 characters]",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "colors = [\"orange\", \"darkgray\", \"r\", \"tab:purple\", \"yellow\"]\n",
    "sns.boxplot(pd.melt(df), x=\"variable\", y=\"value\", palette=colors)\n",
    "sns.despine(top=True, right=True, left=False, bottom=False)\n",
    "\n",
    "# plt.title('Inference time with test set using Raspberry Pi 4', fontsize=18)\n",
    "plt.ylabel('Inference time (s)')\n",
    "plt.xlabel('')\n",
    "plt.xticks(rotation=30)\n",
    "plt.savefig('inference_times.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82a3df8c",
   "metadata": {},
   "source": [
    "### Pipeline time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2c14ca43",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pipeline_df = []\n",
    "for seed in range(0, 50, 10):\n",
    "    df = pd.read_csv(f'./data/results/feature_selection/pipeline/pipeline_times_seed{seed}.csv')\n",
    "    \n",
    "    total_times = df.sum(numeric_only=True)\n",
    "    total_times['Type'] = 'Total Pipeline Time'\n",
    "    \n",
    "    df.loc[-1] = total_times\n",
    "    df.index = df.index + 1\n",
    "    df.sort_index(inplace=True) \n",
    "    \n",
    "    df = pd.melt(df, id_vars=['Type'], value_vars=['Mutual Information', 'Markov Blanket', 'Boruta', 'RFE'])\n",
    "    \n",
    "    all_pipeline_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3258275e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45728 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "\n",
    "ax = sns.barplot(\n",
    "    pd.concat(all_pipeline_df), \n",
    "    y='variable', x='value', \n",
    "    hue='Type', \n",
    "    errorbar=\"sd\", capsize=.4, \n",
    "    err_kws={\"color\": \".33\", \"linewidth\": 1.5}, \n",
    "    palette=[\"orange\", \"olivedrab\", \"yellow\", \"gray\", 'tab:purple'], \n",
    "    width=0.9\n",
    ")\n",
    "sns.despine(top=True, right=True, left=False, bottom=False)\n",
    "sns.move_legend(ax, 'upper right', title=None, frameon=True)\n",
    "\n",
    "plt.title('Pipeline time', fontsize=18)\n",
    "plt.setp(ax.get_legend().get_texts(), fontsize='11')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Time (s)')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
