{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "47dbe264",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings \n",
    "warnings.filterwarnings('ignore') \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler, OrdinalEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "99532c99",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>DGN</th>\n",
       "      <th>PRE4</th>\n",
       "      <th>PRE5</th>\n",
       "      <th>PRE6</th>\n",
       "      <th>PRE7</th>\n",
       "      <th>PRE8</th>\n",
       "      <th>PRE9</th>\n",
       "      <th>PRE10</th>\n",
       "      <th>PRE11</th>\n",
       "      <th>PRE14</th>\n",
       "      <th>PRE17</th>\n",
       "      <th>PRE19</th>\n",
       "      <th>PRE25</th>\n",
       "      <th>PRE30</th>\n",
       "      <th>PRE32</th>\n",
       "      <th>AGE</th>\n",
       "      <th>Risk1Yr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>DGN2</td>\n",
       "      <td>2.88</td>\n",
       "      <td>2.16</td>\n",
       "      <td>PRZ1</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>T</td>\n",
       "      <td>OC14</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>60</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>DGN3</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1.88</td>\n",
       "      <td>PRZ0</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>OC12</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>51</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>DGN3</td>\n",
       "      <td>2.76</td>\n",
       "      <td>2.08</td>\n",
       "      <td>PRZ1</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>OC11</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>59</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>DGN3</td>\n",
       "      <td>3.68</td>\n",
       "      <td>3.04</td>\n",
       "      <td>PRZ0</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>OC11</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>54</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>DGN3</td>\n",
       "      <td>2.44</td>\n",
       "      <td>0.96</td>\n",
       "      <td>PRZ2</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>T</td>\n",
       "      <td>OC11</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>F</td>\n",
       "      <td>73</td>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id   DGN  PRE4  PRE5  PRE6 PRE7 PRE8 PRE9 PRE10 PRE11 PRE14 PRE17 PRE19  \\\n",
       "0   1  DGN2  2.88  2.16  PRZ1    F    F    F     T     T  OC14     F     F   \n",
       "1   2  DGN3  3.40  1.88  PRZ0    F    F    F     F     F  OC12     F     F   \n",
       "2   3  DGN3  2.76  2.08  PRZ1    F    F    F     T     F  OC11     F     F   \n",
       "3   4  DGN3  3.68  3.04  PRZ0    F    F    F     F     F  OC11     F     F   \n",
       "4   5  DGN3  2.44  0.96  PRZ2    F    T    F     T     T  OC11     F     F   \n",
       "\n",
       "  PRE25 PRE30 PRE32  AGE Risk1Yr  \n",
       "0     F     T     F   60       F  \n",
       "1     F     T     F   51       F  \n",
       "2     F     T     F   59       F  \n",
       "3     F     F     F   54       F  \n",
       "4     F     T     F   73       T  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = '/Users/kaanguney.keklikci/Desktop/Erasmus+/Heidelberg/data/ThoraricSurgery.csv'\n",
    "df = pd.read_csv(filename)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c6014eae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('id',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b82b994d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DGN         object\n",
       "PRE4       float64\n",
       "PRE5       float64\n",
       "PRE6        object\n",
       "PRE7        object\n",
       "PRE8        object\n",
       "PRE9        object\n",
       "PRE10       object\n",
       "PRE11       object\n",
       "PRE14       object\n",
       "PRE17       object\n",
       "PRE19       object\n",
       "PRE25       object\n",
       "PRE30       object\n",
       "PRE32       object\n",
       "AGE          int64\n",
       "Risk1Yr     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a97006e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 7008 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(df.DGN,palette='Set3')\n",
    "sns.despine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b25a11b2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 6376 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dgn_types = ['DGN1','DGN8', 'DGN5', 'DGN6']\n",
    "df = df[~df.DGN.isin(dgn_types)]\n",
    "\n",
    "sns.countplot(df.DGN,palette='Set3')\n",
    "sns.despine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "19387a02",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = df.drop('Risk1Yr',axis=1), df.Risk1Yr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01b55142",
   "metadata": {},
   "source": [
    "### Training, validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "dba575cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X, \n",
    "                                                 y,\n",
    "                                                 test_size=.33,\n",
    "                                                 random_state=42,\n",
    "                                                 shuffle=True,\n",
    "                                                 stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "4b3a5041",
   "metadata": {},
   "outputs": [],
   "source": [
    "keepdims = ['DGN','PRE4','PRE5']\n",
    "X_train, X_test = X_train[keepdims], X_test[keepdims]\n",
    "\n",
    "X_train = X_train.reset_index(drop=True)\n",
    "X_test = X_test.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5893ec4e",
   "metadata": {},
   "source": [
    "### Standardize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "04212592",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler()"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train[keepdims[1:]])\n",
    "scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "de7a4ffd",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.append(X_train.DGN.to_numpy().reshape(-1,1), scaler.transform(X_train[keepdims[1:]]), axis=1)\n",
    "X_test = np.append(X_test.DGN.to_numpy().reshape(-1,1), scaler.transform(X_test[keepdims[1:]]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2bd71c25",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_train = pd.DataFrame(X_train, columns=keepdims)\n",
    "X_test = pd.DataFrame(X_test, columns=keepdims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "e4dace09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DGN</th>\n",
       "      <th>PRE4</th>\n",
       "      <th>PRE5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>0.469764</td>\n",
       "      <td>-0.149318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>1.609505</td>\n",
       "      <td>-0.052993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>0.378584</td>\n",
       "      <td>-0.175005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DGN2</td>\n",
       "      <td>-0.35085</td>\n",
       "      <td>-0.207114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>1.575313</td>\n",
       "      <td>-0.083496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    DGN      PRE4      PRE5\n",
       "0  DGN3  0.469764 -0.149318\n",
       "1  DGN3  1.609505 -0.052993\n",
       "2  DGN3  0.378584 -0.175005\n",
       "3  DGN2  -0.35085 -0.207114\n",
       "4  DGN3  1.575313 -0.083496"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "7543c9a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DGN</th>\n",
       "      <th>PRE4</th>\n",
       "      <th>PRE5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>1.427147</td>\n",
       "      <td>-0.097945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DGN2</td>\n",
       "      <td>0.150636</td>\n",
       "      <td>-0.165373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>1.974223</td>\n",
       "      <td>-0.075469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DGN3</td>\n",
       "      <td>-0.852337</td>\n",
       "      <td>-0.232801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DGN4</td>\n",
       "      <td>2.156581</td>\n",
       "      <td>0.011225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    DGN      PRE4      PRE5\n",
       "0  DGN3  1.427147 -0.097945\n",
       "1  DGN2  0.150636 -0.165373\n",
       "2  DGN3  1.974223 -0.075469\n",
       "3  DGN3 -0.852337 -0.232801\n",
       "4  DGN4  2.156581  0.011225"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "2ce29c1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# end of preprocessing"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
