LNP_tsfresh_efficient_gbm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GBMs for t=1 to t=20\n",
    "* GBM grid search using tsfresh time-series features computed in 'LNP_tsfresh_efficient_extract_select_PCA_regression.py'\n",
    "* fit best model from grid search\n",
    "* seperate model for each prediction model (classification or regression) and each cross-validation fold\n",
    "* save out predictions on the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import cv2\n",
    "import xgboost as xgb\n",
    "\n",
    "from scipy.stats import uniform, randint\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, GridSearchCV, KFold, RandomizedSearchCV, train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  import sys\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:11: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "n_time = 20\n",
    "n_latent = 32\n",
    "fold = 'fold5'\n",
    "fit_method = 'regress'\n",
    "\n",
    "X_train_pca = pd.read_csv('/scratch-shared/phil/LNP/LNP_data_09/tsfresh_efficient_pca_train_features_' + fit_method + '_' + fold + '.csv')\n",
    "X_train_pca = X_train_pca.as_matrix()\n",
    "X_train_pca = X_train_pca[:,1:]\n",
    "\n",
    "X_test_pca = pd.read_csv('/scratch-shared/phil/LNP/LNP_data_09/tsfresh_efficient_pca_test_features_' + fit_method + '_' + fold + '.csv')\n",
    "X_test_pca = X_test_pca.as_matrix()\n",
    "X_test_pca = X_test_pca[:,1:]\n",
    "\n",
    "train_ids = np.load('/scratch-shared/phil/LNP/LNP_data_09/train_cell_ids_' + fit_method + '_' + fold + '.npy')\n",
    "\n",
    "train_y = np.load('/scratch-shared/phil/LNP/LNP_data_09/train_cell_gfp_' + fit_method + '_' + fold + '.npy')\n",
    "test_y = np.load('/scratch-shared/phil/LNP/LNP_data_09/test_cell_gfp_' + fit_method + '_' + fold + '.npy')\n",
    "\n",
    "train_index = []\n",
    "valid_index = []\n",
    "\n",
    "for i in range(len(train_ids)):\n",
    "    s0 = train_ids[i].split('train/')\n",
    "    s1 = s0[1].split('_')[0]\n",
    "    if s1 == fold:\n",
    "        valid_index.append(i)\n",
    "    else:\n",
    "        train_index.append(i)\n",
    "\n",
    "X_train_pca_train = X_train_pca[train_index]\n",
    "train_y_train = train_y[train_index]\n",
    "\n",
    "X_train_pca_valid = X_train_pca[valid_index]\n",
    "train_y_valid = train_y[valid_index]\n",
    "\n",
    "# getting positive class weights for when in classification mode\n",
    "if fit_method == 'classify':\n",
    "    class_gfp = train_y_train.astype('int64')\n",
    "    scale_pos = (len(class_gfp) - np.sum(class_gfp)) / np.sum(class_gfp)\n",
    "    print('pos weight = ' + str(scale_pos))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_names = ['colsamp', 'gamma', 'lr', 'max_d', 'n_est', 'subsamp', 'min_rmse']\n",
    "\n",
    "for repl in range(200):\n",
    "    colsamp = np.random.uniform(0.3, 0.7)\n",
    "    gamma = np.random.uniform(0.0, 0.5)\n",
    "    lr = np.random.uniform(0.03, 0.3)\n",
    "    max_d = np.random.randint(2, 6)\n",
    "    n_est = 300 # now using early stopping so this is just an upper limit\n",
    "    subsamp = np.random.uniform(0.4, 0.6)\n",
    "    \n",
    "    if fit_method == 'classify':\n",
    "        xgb_model = xgb.XGBClassifier(objective = \"binary:logistic\",\n",
    "                                      colsample_bytree = colsamp,\n",
    "                                      gamma = gamma,\n",
    "                                      learning_rate = lr,\n",
    "                                      max_depth = max_d,\n",
    "                                      n_estimators = n_est,\n",
    "                                      subsample = subsamp,\n",
    "                                      scale_pos_weight = scale_pos,\n",
    "                                      n_jobs = 3)\n",
    "        eval_metric = \"logloss\"\n",
    "    else:\n",
    "        xgb_model = xgb.XGBRegressor(objective ='reg:squarederror',\n",
    "                                     colsample_bytree = colsamp,\n",
    "                                     gamma = gamma,\n",
    "                                     learning_rate = lr,\n",
    "                                     max_depth = max_d,\n",
    "                                     n_estimators = n_est,\n",
    "                                     subsample = subsamp,\n",
    "                                     n_jobs = 3)\n",
    "        eval_metric = \"rmse\"\n",
    "    \n",
    "    eval_set = [(X_train_pca_valid, train_y_valid)]\n",
    "        \n",
    "    xgb_model.fit(X_train_pca_train, train_y_train, early_stopping_rounds=10, eval_metric=eval_metric, \n",
    "                  eval_set=eval_set, verbose=0)\n",
    "    \n",
    "    y_pred = xgb_model.predict(X_train_pca_valid)\n",
    "    \n",
    "    val_losses = xgb_model.evals_result()\n",
    "    \n",
    "    res = np.zeros((1,7))\n",
    "    res[0, 0] = colsamp\n",
    "    res[0, 1] = gamma\n",
    "    res[0, 2] = lr\n",
    "    res[0, 3] = max_d\n",
    "    res[0, 4] = n_est\n",
    "    res[0, 5] = subsamp\n",
    "    if fit_method == 'classify':\n",
    "        res[0, 6] = min(val_losses['validation_0']['logloss'])\n",
    "    else:\n",
    "        res[0, 6] = min(val_losses['validation_0']['rmse'])\n",
    "    \n",
    "    if repl == 0:\n",
    "        res_df = pd.DataFrame(res, columns=col_names)\n",
    "    else:\n",
    "        df = pd.DataFrame(res, columns=col_names)\n",
    "        res_df = res_df.append(df, ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Print out 5 best models selected via grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>colsamp</th>\n",
       "      <th>gamma</th>\n",
       "      <th>lr</th>\n",
       "      <th>max_d</th>\n",
       "      <th>n_est</th>\n",
       "      <th>subsamp</th>\n",
       "      <th>min_rmse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>149</td>\n",
       "      <td>0.672299</td>\n",
       "      <td>0.481209</td>\n",
       "      <td>0.105425</td>\n",
       "      <td>4.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.444650</td>\n",
       "      <td>0.566725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>99</td>\n",
       "      <td>0.533801</td>\n",
       "      <td>0.300534</td>\n",
       "      <td>0.056640</td>\n",
       "      <td>5.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.428597</td>\n",
       "      <td>0.567073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>143</td>\n",
       "      <td>0.599805</td>\n",
       "      <td>0.336235</td>\n",
       "      <td>0.040526</td>\n",
       "      <td>5.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.446401</td>\n",
       "      <td>0.568722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>102</td>\n",
       "      <td>0.671026</td>\n",
       "      <td>0.403088</td>\n",
       "      <td>0.040559</td>\n",
       "      <td>5.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.505051</td>\n",
       "      <td>0.571376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>96</td>\n",
       "      <td>0.609975</td>\n",
       "      <td>0.296452</td>\n",
       "      <td>0.165932</td>\n",
       "      <td>4.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.510807</td>\n",
       "      <td>0.571583</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index   colsamp     gamma        lr  max_d  n_est   subsamp  min_rmse\n",
       "0    149  0.672299  0.481209  0.105425    4.0  300.0  0.444650  0.566725\n",
       "1     99  0.533801  0.300534  0.056640    5.0  300.0  0.428597  0.567073\n",
       "2    143  0.599805  0.336235  0.040526    5.0  300.0  0.446401  0.568722\n",
       "3    102  0.671026  0.403088  0.040559    5.0  300.0  0.505051  0.571376\n",
       "4     96  0.609975  0.296452  0.165932    4.0  300.0  0.510807  0.571583"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if fit_method == 'classify':\n",
    "    res_df = res_df.sort_values(ascending=True, by='min_val_loss')\n",
    "else:\n",
    "    res_df = res_df.sort_values(ascending=True, by='min_rmse')\n",
    "res_df = res_df.reset_index()\n",
    "res_df[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train best model from above and make predictions on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse for test data\n",
      "0.626\n"
     ]
    }
   ],
   "source": [
    "colsamp = res_df['colsamp'][0]\n",
    "gamma = res_df['gamma'][0]\n",
    "lr = res_df['lr'][0]\n",
    "max_d = int(res_df['max_d'][0])\n",
    "n_est = int(res_df['n_est'][0])\n",
    "subsamp = res_df['subsamp'][0]\n",
    "\n",
    "if fit_method == 'classify':\n",
    "    xgb_model = xgb.XGBClassifier(objective = \"binary:logistic\",\n",
    "                                  colsample_bytree = colsamp,\n",
    "                                  gamma = gamma,\n",
    "                                  learning_rate = lr,\n",
    "                                  max_depth = max_d,\n",
    "                                  n_estimators = n_est,\n",
    "                                  subsample = subsamp,\n",
    "                                  scale_pos_weight = scale_pos,\n",
    "                                  n_jobs = 3)\n",
    "    eval_metric = \"logloss\"\n",
    "else:    \n",
    "    xgb_model = xgb.XGBRegressor(objective ='reg:squarederror',\n",
    "                                 colsample_bytree = colsamp,\n",
    "                                 gamma = gamma,\n",
    "                                 learning_rate = lr,\n",
    "                                 max_depth = max_d,\n",
    "                                 n_estimators = n_est,\n",
    "                                 subsample = subsamp,\n",
    "                                 n_jobs = 3)\n",
    "    eval_metric = \"rmse\"\n",
    "\n",
    "eval_set = [(X_train_pca_valid, train_y_valid)]\n",
    "\n",
    "xgb_model.fit(X_train_pca_train, train_y_train, early_stopping_rounds=10, eval_metric=eval_metric, \n",
    "              eval_set=eval_set, verbose=0)\n",
    "\n",
    "y_pred = xgb_model.predict(X_test_pca)\n",
    "np.save('/scratch-shared/phil/LNP/LNP_data_09/tsf_efficient_test_pred_' + fit_method + '_' + fold + '.npy', y_pred)\n",
    "\n",
    "\n",
    "print('rmse for test data')\n",
    "print(np.round(np.sqrt(mean_squared_error(test_y, y_pred)), decimals=3))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}