{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\" use smaller learning rate for gradient descent or increase batch size \"\"\"\n",
    "\n",
    "import os \n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n",
    "import time \n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.compat.v1.disable_eager_execution() \n",
    "import tensorflow_probability as tfp\n",
    "import tensorflow.python.util.deprecation as deprecation\n",
    "deprecation._PRINT_DEPRECATION_WARNINGS = False\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from data_loader import load_data\n",
    "from data_preprocesser import preprocess_data\n",
    "from maf import MAF \n",
    "from experiment import Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data successfully loaded...\n",
      "\n",
      "Data successfully preprocessed...\n",
      "\n",
      "TensorFlow version: 2.5.0\n",
      "Number of dimensions: 37\n",
      "Learning rate: 0.0001\n",
      "\n",
      "Successfully created model...\n",
      "\n",
      "Optimizer: Adam_2\n",
      "Optimizer and loss successfully defined...\n",
      "\n",
      "Iteration 0: 52.72296142578125\n",
      "Iteration 10000: -12.123727798461914\n",
      "Iteration 20000: -31.257366180419922\n",
      "Iteration 30000: -19.52370834350586\n",
      "Iteration 40000: -34.72262954711914\n",
      "Iteration 50000: -37.22956085205078\n",
      "Iteration 60000: -31.837202072143555\n",
      "Iteration 70000: -36.162353515625\n",
      "Iteration 80000: -40.44646453857422\n",
      "Iteration 90000: -39.50082015991211\n",
      "\n",
      "Training completed...\n",
      "Training time: 508.4956090450287 seconds\n",
      "Training finished...\n",
      "\n",
      "Displaying results...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 47644 characters]\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizer: RMSProp\n",
      "Optimizer and loss successfully defined...\n",
      "\n",
      "Iteration 0: 52.72296142578125\n",
      "Iteration 10000: -8.40821361541748\n",
      "Iteration 20000: -24.88846778869629\n",
      "Iteration 30000: -13.85255241394043\n",
      "Iteration 40000: -30.42304229736328\n",
      "Iteration 50000: -33.980743408203125\n",
      "Iteration 60000: -26.754453659057617\n",
      "Iteration 70000: -35.094207763671875\n",
      "Iteration 80000: -34.587615966796875\n",
      "Iteration 90000: -27.752574920654297\n",
      "\n",
      "Training completed...\n",
      "Training time: 565.5382442474365 seconds\n",
      "Training finished...\n",
      "\n",
      "Displaying results...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 49336 characters]\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizer: GradientDescent\n",
      "Optimizer and loss successfully defined...\n",
      "\n",
      "Iteration 0: 52.72296142578125\n",
      "Iteration 10000: 31.858509063720703\n",
      "Iteration 20000: 14.227487564086914\n",
      "Iteration 30000: 5.67965841293335\n",
      "Iteration 40000: -4.409252643585205\n",
      "Iteration 50000: nan\n",
      "Iteration 60000: nan\n",
      "Iteration 70000: nan\n",
      "Iteration 80000: nan\n",
      "Iteration 90000: nan\n",
      "\n",
      "Training completed...\n",
      "Training time: 490.9801027774811 seconds\n",
      "Training finished...\n",
      "\n",
      "Displaying results...\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 27452 characters]\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train(session, loss, optimizer, steps=int(1e5)):\n",
    "    \n",
    "    \"\"\" optimize for all dimensions \"\"\"\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    recorded_steps = []\n",
    "    recorded_losses = []\n",
    "    for i in range(steps):\n",
    "        _, loss_per_iteration = session.run([optimizer, loss])\n",
    "        if i % 100 == 0:\n",
    "            recorded_steps.append(i)\n",
    "            recorded_losses.append(loss_per_iteration)\n",
    "        if i % int(1e4) == 0:\n",
    "            print('Iteration {iteration}: {loss}'.format(iteration=i,loss=loss_per_iteration))\n",
    "    print('\\nTraining completed...')\n",
    "    print(f'Training time: {time.time() - start_time} seconds')\n",
    "    return recorded_losses\n",
    "\n",
    "\n",
    "def plot_results(recorded_losses):\n",
    "    \n",
    "    \"\"\" plot loss \"\"\"\n",
    "    print('Displaying results...')\n",
    "    fig = plt.figure(figsize=(10,5))\n",
    "    x = np.arange(len(recorded_losses))\n",
    "    y = recorded_losses\n",
    "    m, b = np.polyfit(x, y, 1) \n",
    "    plt.scatter(x, y, s=10, alpha=0.3)\n",
    "    plt.plot(x, m*x+b, c=\"r\")\n",
    "    plt.title('Loss per 100 iteration')\n",
    "    plt.xlabel('Iteration')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "def main():\n",
    "    \n",
    "    \"\"\" load data \"\"\"\n",
    "\n",
    "    filename = 'prostate.xls'\n",
    "    directory = '/Users/kaanguney.keklikci/Data/'\n",
    "\n",
    "    loader = load_data(filename, directory)\n",
    "    loader.create_directory(directory)\n",
    "    data = loader.read_data(directory, filename)\n",
    "    print('Data successfully loaded...\\n')\n",
    "    \n",
    "    \"\"\" preprocess data \"\"\"\n",
    "\n",
    "    fillna_vals = ['sz', 'sg', 'wt']\n",
    "    dropna_vals = ['ekg', 'age']\n",
    "    drop_vals = ['patno', 'sdate']\n",
    "\n",
    "    preprocesser = preprocess_data(StandardScaler(), fillna_vals, dropna_vals, drop_vals)\n",
    "    data = preprocesser.dropna_features(data)\n",
    "    data = preprocesser.impute(data)\n",
    "    data = preprocesser.drop_features(data)\n",
    "    data = preprocesser.encode_categorical(data)\n",
    "    data = preprocesser.scale(data)\n",
    "    print('Data successfully preprocessed...\\n')\n",
    "    \n",
    "    \"\"\" set MAF parameters \"\"\"\n",
    "\n",
    "    batch_size = 32\n",
    "    dtype = np.float32\n",
    "    tf_version = tf.__version__\n",
    "    params = 2\n",
    "    hidden_units = [512,512]\n",
    "    base_dist = tfp.distributions.Normal(loc=0., scale=1., name=\"gaussian\")\n",
    "    dims = data.shape[1]\n",
    "    learning_rate = 1e-4\n",
    "    steps = 1e4\n",
    "\n",
    "    \"\"\" initialize samples \"\"\"\n",
    "\n",
    "    maf = MAF(dtype, tf_version, batch_size, params, hidden_units, base_dist, dims)\n",
    "\n",
    "    dims = maf.get_dims(data)\n",
    "    samples = maf.create_tensor(data)\n",
    "    print(f'TensorFlow version: {maf.tf_version}')\n",
    "    print(f'Number of dimensions: {maf.dims}')\n",
    "    print(f'Learning rate: {learning_rate}\\n')\n",
    "    \n",
    "    \"\"\" initialize MAF \"\"\"\n",
    "\n",
    "    maf = maf.make_maf(data)\n",
    "    print('Successfully created model...\\n')\n",
    "    \n",
    "    \"\"\" initialize loss and optimizer \"\"\"\n",
    "\n",
    "    loss = -tf.reduce_mean(maf.log_prob(samples))\n",
    "    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "    \n",
    "    experiment = Experiment(optimizer, learning_rate, loss, steps)\n",
    "    \n",
    "    keywords = ['adam', 'rmsprop', 'sgd']\n",
    "    \n",
    "    for keyword in keywords:\n",
    "    \n",
    "        session = tf.compat.v1.Session()\n",
    "        tf.compat.v1.set_random_seed(42)\n",
    "        experiment.change_optimizer(learning_rate, loss, keyword=keyword)\n",
    "        optimizer = experiment.get_optimizer()\n",
    "        session.run(tf.compat.v1.global_variables_initializer())\n",
    "        print(f'Optimizer: {optimizer.name}')\n",
    "        print('Optimizer and loss successfully defined...\\n')\n",
    "\n",
    "        \"\"\" start training \"\"\"\n",
    "        recorded_losses = train(session, loss, optimizer)\n",
    "        print('Training finished...\\n')\n",
    "\n",
    "        \"\"\" display results \"\"\"\n",
    "        plot_results(recorded_losses)\n",
    "    \n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
