4b-linear.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "Qb-B_JJx2RLU",
   "metadata": {
    "id": "Qb-B_JJx2RLU"
   },
   "source": [
    "# Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b3017762",
   "metadata": {
    "id": "b3017762"
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'torch'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/t8/rn08vqd96sq52_vd02c7vqn00000gn/T/ipykernel_74181/4089718988.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mf1_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mroc_auc_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrnn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpad_sequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'torch'"
     ]
    }
   ],
   "source": [
    "# --- Dependencies ---\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import f1_score, accuracy_score, roc_auc_score\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "CP-89d70jT_0",
   "metadata": {
    "id": "CP-89d70jT_0"
   },
   "outputs": [],
   "source": [
    "with open('train.pkl', 'rb') as f:\n",
    "  train = pickle.load(f)\n",
    "with open('validation.pkl', 'rb') as f:\n",
    "  validation = pickle.load(f)\n",
    "with open('test.pkl', 'rb') as f:\n",
    "  test = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26f85e8a",
   "metadata": {
    "id": "26f85e8a"
   },
   "outputs": [],
   "source": [
    "X_train = [sequence for sequence, target in train]\n",
    "X_val = [sequence for sequence, target in validation]\n",
    "X_test = [sequence for sequence, target in test]\n",
    "\n",
    "Y_train = [target for sequence, target in train]\n",
    "Y_val = [target for sequence, target in validation]\n",
    "Y_test = [target for sequence, target in test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "sPxKIO6mnyu8",
   "metadata": {
    "id": "sPxKIO6mnyu8"
   },
   "outputs": [],
   "source": [
    "INPUT_SIZE = len(X_train[0][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "M3Dq7KaM6bAs",
   "metadata": {
    "id": "M3Dq7KaM6bAs"
   },
   "source": [
    "# Dataset / DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c8c0d38c",
   "metadata": {
    "id": "c8c0d38c"
   },
   "outputs": [],
   "source": [
    "class ReviewDataset(Dataset):\n",
    "    def __init__(self, X, y):\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return torch.from_numpy(self.X[idx]).float(), torch.tensor(self.y[idx]).float()\n",
    "\n",
    "train_dataset = ReviewDataset(X_train, Y_train)\n",
    "val_dataset = ReviewDataset(X_val, Y_val)\n",
    "test_dataset = ReviewDataset(X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "-TJcrHOCv-hd",
   "metadata": {
    "id": "-TJcrHOCv-hd"
   },
   "outputs": [],
   "source": [
    "# Pad the sequences\n",
    "def collate_fn(batch):\n",
    "    inputs = [item[0] for item in batch]\n",
    "    targets = [item[1] for item in batch]\n",
    "    \n",
    "    # Pad sequences so they all have the same length within one batch\n",
    "    inputs_padded = pad_sequence(inputs, batch_first=True, padding_value=0)\n",
    "\n",
    "    return inputs_padded, torch.stack(targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "xtB9m21VwIr2",
   "metadata": {
    "id": "xtB9m21VwIr2"
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "train_loader = DataLoader(train_dataset, collate_fn=collate_fn, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=1)\n",
    "test_loader = DataLoader(test_dataset, batch_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "O8HsoSk_6fla",
   "metadata": {
    "id": "O8HsoSk_6fla"
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0896eab",
   "metadata": {
    "id": "f0896eab"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'nn' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/t8/rn08vqd96sq52_vd02c7vqn00000gn/T/ipykernel_74181/2530428848.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mFeedforwardModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFeedforwardModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhidden_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'nn' is not defined"
     ]
    }
   ],
   "source": [
    "class LinearModel(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, dropout):\n",
    "        super(LinearModel, self).__init__()\n",
    "\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size) \n",
    "        self.fc2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, 2)  # Two output nodes for λ and k\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # flatten the input\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        # layers\n",
    "        x = self.dropout(F.relu(self.fc1(x)))\n",
    "        x = self.dropout(F.relu(self.fc2(x)))\n",
    "        out = self.out(x)\n",
    "\n",
    "        # output\n",
    "        λ, k = torch.exp(out[:, 0]), torch.exp(out[:, 1])  # exp to ensure positive values\n",
    "        return λ, k\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "32685941",
   "metadata": {
    "id": "32685941"
   },
   "outputs": [],
   "source": [
    "def weibull_survival_function(λ, k, t):\n",
    "  # small constant to prevent division by zero\n",
    "  epsilon = 1e-7\n",
    "  λ = torch.clamp(λ, min=epsilon)\n",
    "  return torch.exp(-(t / λ) ** k)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "CeRMDRtp6jeN",
   "metadata": {
    "id": "CeRMDRtp6jeN"
   },
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "QhZY_FESxtou",
   "metadata": {
    "id": "QhZY_FESxtou"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c96688ff",
   "metadata": {
    "id": "c96688ff"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score, fbeta_score, accuracy_score, precision_score, recall_score\n",
    "\n",
    "def calculate_metrics(model, loader, device, threshold, metrics, print_results=False):\n",
    "    model.eval()\n",
    "    all_preds = []\n",
    "    all_targets = []\n",
    "    with torch.no_grad():\n",
    "        for inputs, targets in loader:\n",
    "            inputs, targets = inputs.to(device), targets.to(device)\n",
    "            λ, k = model(inputs)\n",
    "            review_time = targets[:, 0]\n",
    "            remembered = targets[:, 1]\n",
    "            predicted_survival = weibull_survival_function(λ, k, review_time)\n",
    "\n",
    "            preds = predicted_survival\n",
    "            all_preds.append(preds.cpu().numpy())\n",
    "            all_targets.append(remembered.cpu().numpy())\n",
    "\n",
    "    # Flatten the lists of arrays for computing metrics\n",
    "    all_preds = np.concatenate(all_preds)\n",
    "    all_targets = np.concatenate(all_targets)\n",
    "    \n",
    "    threshold_preds = (all_preds > threshold).astype(int)\n",
    "\n",
    "    results = {}\n",
    "    for metric in metrics:\n",
    "        if metric == \"roc_auc\":\n",
    "            result = roc_auc_score(all_targets, all_preds)\n",
    "        elif metric == \"f2\":\n",
    "            result = fbeta_score(all_targets, threshold_preds, beta=2)\n",
    "        elif metric == \"accuracy\":\n",
    "            result = accuracy_score(all_targets, threshold_preds)\n",
    "        elif metric == \"precision\":\n",
    "            result = precision_score(all_targets, threshold_preds)\n",
    "        elif metric == \"recall\":\n",
    "            result = recall_score(all_targets, threshold_preds)\n",
    "        else:\n",
    "            raise ValueError(f\"Unsupported metric: {metric}\")\n",
    "        \n",
    "        results[metric] = result\n",
    "    if print_results:\n",
    "        print(results)\n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "xbfR2UV_MQdB",
   "metadata": {
    "id": "xbfR2UV_MQdB"
   },
   "outputs": [],
   "source": [
    "def log_scores(epoch_scores, score_dict=None):\n",
    "    \"\"\"\n",
    "    This function takes the epoch scores and a dictionary of previous scores (or None),\n",
    "    and returns an updated dictionary of scores.\n",
    "    \"\"\"\n",
    "\n",
    "    if score_dict is None:\n",
    "        # Initialize score_dict with empty lists if it's None\n",
    "        score_dict = {key: [] for key in epoch_scores.keys()}\n",
    "\n",
    "    for key in epoch_scores.keys():\n",
    "        if key in score_dict:\n",
    "            score_dict[key].append(epoch_scores[key])\n",
    "        else:\n",
    "            score_dict[key] = [epoch_scores[key]]\n",
    "\n",
    "    return score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a2e40198",
   "metadata": {
    "id": "a2e40198"
   },
   "outputs": [],
   "source": [
    "def train(*, epochs = 5, hidden_size = 128, num_layers = 2, lr = 0.001, dropout = 0.2, weight_decay = 0):\n",
    "  model = GRUModel(INPUT_SIZE, hidden_size, num_layers, dropout)  # Adjust the parameters as needed\n",
    "  model = model.to(device)\n",
    "  \n",
    "  optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)  # Adjust the learning rate as needed\n",
    "  criterion = nn.BCELoss()\n",
    "\n",
    "  scores = {}\n",
    "  for epoch in range(epochs):\n",
    "      model.train()\n",
    "\n",
    "      train_loss = 0.0\n",
    "\n",
    "      progress = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs}\")\n",
    "      for inputs, targets in progress:\n",
    "          inputs, targets = inputs.to(device), targets.to(device)\n",
    "          \n",
    "          λ, k = model(inputs)\n",
    "          review_time = targets[:, 0]\n",
    "          remembered = targets[:, 1]\n",
    "          \n",
    "          predicted_survival = weibull_survival_function(λ, k, review_time)\n",
    "\n",
    "          loss = criterion(predicted_survival, remembered)\n",
    "\n",
    "          optimizer.zero_grad()\n",
    "\n",
    "          # clip grads\n",
    "          torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1)\n",
    "\n",
    "          loss.backward() \n",
    "          optimizer.step()\n",
    "\n",
    "          train_loss += loss.item()\n",
    "          progress.set_postfix({'loss': train_loss / (progress.n + 1)})\n",
    "\n",
    "      epoch_loss = train_loss / len(train_loader)\n",
    "      print(epoch_loss)\n",
    "\n",
    "      if epoch % 2 == 0:\n",
    "        #epoch_scores = calculate_metrics(model, train_loader, device, metrics=[\"roc_auc\", \"recall\", \"precision\"], print_results=True)\n",
    "        epoch_scores = calculate_metrics(model, val_loader, device, threshold=0.6, metrics=[\"roc_auc\"], print_results=True)\n",
    "        #epoch_scores = calculate_metrics(model, val_loader, device, threshold=0.7, metrics=[\"roc_auc\", \"recall\", \"precision\"], print_results=True)\n",
    "        # epoch_scores = calculate_metrics(model, val_loader, device, threshold=0.8, metrics=[\"roc_auc\", \"recall\", \"precision\"], print_results=True)\n",
    "        # epoch_scores = calculate_metrics(model, test_loader, device, metrics=[\"roc_auc\", \"recall\", \"precision\"], print_results=True)\n",
    "        scores = log_scores(epoch_scores, scores)\n",
    "\n",
    "  return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "_tsqdDUCODlG",
   "metadata": {
    "id": "_tsqdDUCODlG"
   },
   "outputs": [],
   "source": [
    "def print_max_values(input_dict):\n",
    "    for outer_key, inner_dict in input_dict.items():\n",
    "        print(outer_key)\n",
    "        for inner_key, values_list in inner_dict.items():\n",
    "            max_value_index = np.argmax(values_list)\n",
    "            max_value = values_list[max_value_index]\n",
    "            print(f'{inner_key}: {round(max_value, 5)} (at {max_value_index})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "jLP83i1_48Jc",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jLP83i1_48Jc",
    "outputId": "5255b37c-1696-4c6f-aa2f-cd679cbbbcea"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/40: 100%|██████████| 416/416 [00:03<00:00, 107.77it/s, loss=0.401]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3908064153690178\n",
      "{'roc_auc': 0.5886040648046369}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/40: 100%|██████████| 416/416 [00:03<00:00, 124.99it/s, loss=0.371]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3682910423869124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/40: 100%|██████████| 416/416 [00:03<00:00, 125.15it/s, loss=0.374]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36780605762480545\n",
      "{'roc_auc': 0.6289593682624282}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/40: 100%|██████████| 416/416 [00:03<00:00, 126.74it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36481593251944733\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/40: 100%|██████████| 416/416 [00:03<00:00, 124.33it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3631306928940691\n",
      "{'roc_auc': 0.6687022763819259}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/40: 100%|██████████| 416/416 [00:03<00:00, 112.72it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36129565784134543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/40: 100%|██████████| 416/416 [00:03<00:00, 117.53it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3622849896693459\n",
      "{'roc_auc': 0.6673943108391529}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/40: 100%|██████████| 416/416 [00:03<00:00, 123.73it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36174925284173626\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/40: 100%|██████████| 416/416 [00:03<00:00, 116.11it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3613704265668415\n",
      "{'roc_auc': 0.6809346897903668}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/40: 100%|██████████| 416/416 [00:03<00:00, 123.11it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36017938868071026\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/40: 100%|██████████| 416/416 [00:03<00:00, 124.91it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3603791996406821\n",
      "{'roc_auc': 0.6903872687004824}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/40: 100%|██████████| 416/416 [00:03<00:00, 109.09it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3592173269806573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/40: 100%|██████████| 416/416 [00:03<00:00, 120.80it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35984383763458866\n",
      "{'roc_auc': 0.6858648392115675}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/40: 100%|██████████| 416/416 [00:03<00:00, 124.83it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3600102938138522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/40: 100%|██████████| 416/416 [00:03<00:00, 110.61it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3591740599546868\n",
      "{'roc_auc': 0.6879617428233138}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/40: 100%|██████████| 416/416 [00:03<00:00, 121.88it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3585169714374038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/40: 100%|██████████| 416/416 [00:03<00:00, 123.06it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35846707540062756\n",
      "{'roc_auc': 0.6934500608318139}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/40: 100%|██████████| 416/416 [00:03<00:00, 113.58it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35967556963889646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/40: 100%|██████████| 416/416 [00:03<00:00, 121.63it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35880620354929793\n",
      "{'roc_auc': 0.6986427818894068}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/40: 100%|██████████| 416/416 [00:03<00:00, 116.46it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35840271313029987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/40: 100%|██████████| 416/416 [00:03<00:00, 109.05it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35729072185663074\n",
      "{'roc_auc': 0.6860764458564239}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/40: 100%|██████████| 416/416 [00:03<00:00, 122.99it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3578529538443455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/40: 100%|██████████| 416/416 [00:03<00:00, 120.58it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35747941934431976\n",
      "{'roc_auc': 0.6961662910207024}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/40: 100%|██████████| 416/416 [00:03<00:00, 114.51it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35699611350607413\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/40: 100%|██████████| 416/416 [00:03<00:00, 121.02it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35696862598594564\n",
      "{'roc_auc': 0.6975549851100681}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/40: 100%|██████████| 416/416 [00:03<00:00, 125.50it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35653834825811476\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/40: 100%|██████████| 416/416 [00:03<00:00, 111.05it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3561308947391808\n",
      "{'roc_auc': 0.6880608187668592}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/40: 100%|██████████| 416/416 [00:03<00:00, 123.70it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3551270059094979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/40: 100%|██████████| 416/416 [00:03<00:00, 121.72it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3551671480974899\n",
      "{'roc_auc': 0.6922974365832417}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/40: 100%|██████████| 416/416 [00:03<00:00, 110.17it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3544555199332535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 31/40: 100%|██████████| 416/416 [00:03<00:00, 122.79it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3542618492904764\n",
      "{'roc_auc': 0.689850301549662}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 32/40: 100%|██████████| 416/416 [00:03<00:00, 121.56it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35463494825391817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 33/40: 100%|██████████| 416/416 [00:03<00:00, 105.77it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35322609955731493\n",
      "{'roc_auc': 0.6953532974757244}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 34/40: 100%|██████████| 416/416 [00:03<00:00, 122.81it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3531861180224671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 35/40: 100%|██████████| 416/416 [00:03<00:00, 122.12it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3520050071752988\n",
      "{'roc_auc': 0.6889708496557212}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 36/40: 100%|██████████| 416/416 [00:03<00:00, 113.42it/s, loss=0.354]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3517244094624542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 37/40: 100%|██████████| 416/416 [00:03<00:00, 120.89it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.350469437141258\n",
      "{'roc_auc': 0.6903057247140253}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 38/40: 100%|██████████| 416/416 [00:03<00:00, 122.46it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34961223881691694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 39/40: 100%|██████████| 416/416 [00:03<00:00, 106.28it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34950511279301\n",
      "{'roc_auc': 0.6909425832482559}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 40/40: 100%|██████████| 416/416 [00:03<00:00, 107.68it/s, loss=0.35]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3485754510531059\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.69864 (at 9)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/40: 100%|██████████| 416/416 [00:03<00:00, 121.58it/s, loss=0.392]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3918728253875787\n",
      "{'roc_auc': 0.5692096430656626}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/40: 100%|██████████| 416/416 [00:03<00:00, 114.13it/s, loss=0.374]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37015045867659724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/40: 100%|██████████| 416/416 [00:03<00:00, 120.73it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3675766293890774\n",
      "{'roc_auc': 0.6309600499701549}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/40: 100%|██████████| 416/416 [00:03<00:00, 114.06it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36642893067059606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/40: 100%|██████████| 416/416 [00:03<00:00, 110.30it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3638006957391134\n",
      "{'roc_auc': 0.6641957479703702}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/40: 100%|██████████| 416/416 [00:03<00:00, 120.62it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36175307577762467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/40: 100%|██████████| 416/416 [00:03<00:00, 117.14it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36174070928245783\n",
      "{'roc_auc': 0.6749754552600764}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/40: 100%|██████████| 416/416 [00:03<00:00, 121.29it/s, loss=0.372]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36170236768129355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/40: 100%|██████████| 416/416 [00:03<00:00, 121.63it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3608325906097889\n",
      "{'roc_auc': 0.684148338296644}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/40: 100%|██████████| 416/416 [00:03<00:00, 114.16it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36116627865256024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/40: 100%|██████████| 416/416 [00:03<00:00, 111.54it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35990321134718567\n",
      "{'roc_auc': 0.6906212999416145}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/40: 100%|██████████| 416/416 [00:03<00:00, 120.69it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35991487361920566\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/40: 100%|██████████| 416/416 [00:03<00:00, 116.54it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35962947490266883\n",
      "{'roc_auc': 0.68633616345329}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/40: 100%|██████████| 416/416 [00:03<00:00, 117.60it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3591491976896158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/40: 100%|██████████| 416/416 [00:03<00:00, 118.65it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35837494134186554\n",
      "{'roc_auc': 0.6875674776487933}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/40: 100%|██████████| 416/416 [00:03<00:00, 107.31it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35892867991844046\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/40: 100%|██████████| 416/416 [00:03<00:00, 117.19it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3591611950586622\n",
      "{'roc_auc': 0.6855480408241814}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/40: 100%|██████████| 416/416 [00:03<00:00, 117.05it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3581081974821595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/40: 100%|██████████| 416/416 [00:03<00:00, 114.04it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.358430869734058\n",
      "{'roc_auc': 0.6835102566026166}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/40: 100%|██████████| 416/416 [00:03<00:00, 120.37it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3576365092840905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/40: 100%|██████████| 416/416 [00:03<00:00, 119.07it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35711904203232664\n",
      "{'roc_auc': 0.687381557359671}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/40: 100%|██████████| 416/416 [00:03<00:00, 104.30it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3561296099796891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/40: 100%|██████████| 416/416 [00:03<00:00, 118.45it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3555582071511218\n",
      "{'roc_auc': 0.6891486155461979}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/40: 100%|██████████| 416/416 [00:03<00:00, 119.22it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3569485991118619\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/40: 100%|██████████| 416/416 [00:03<00:00, 111.58it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3561312746829711\n",
      "{'roc_auc': 0.6880110769351203}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/40: 100%|██████████| 416/416 [00:03<00:00, 115.54it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3547549583017826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/40: 100%|██████████| 416/416 [00:03<00:00, 116.36it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35420871622717154\n",
      "{'roc_auc': 0.6846098772599916}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/40: 100%|██████████| 416/416 [00:03<00:00, 112.87it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.353168411896779\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/40: 100%|██████████| 416/416 [00:03<00:00, 114.31it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3533674943117568\n",
      "{'roc_auc': 0.6821121849548084}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/40: 100%|██████████| 416/416 [00:03<00:00, 116.15it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35321670487666357\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 31/40: 100%|██████████| 416/416 [00:03<00:00, 108.83it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35187065923729766\n",
      "{'roc_auc': 0.6875903099650014}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 32/40: 100%|██████████| 416/416 [00:03<00:00, 118.69it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3504740116902842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 33/40: 100%|██████████| 416/416 [00:03<00:00, 120.70it/s, loss=0.354]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35079775338705915\n",
      "{'roc_auc': 0.688223091299909}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 34/40: 100%|██████████| 416/416 [00:03<00:00, 110.88it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34985944976170474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 35/40: 100%|██████████| 416/416 [00:03<00:00, 117.16it/s, loss=0.35]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34846898916965496\n",
      "{'roc_auc': 0.6870977842868}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 36/40: 100%|██████████| 416/416 [00:03<00:00, 117.07it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3473340136428865\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 37/40: 100%|██████████| 416/416 [00:03<00:00, 110.26it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34673692379146814\n",
      "{'roc_auc': 0.680608513844538}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 38/40: 100%|██████████| 416/416 [00:03<00:00, 119.09it/s, loss=0.349]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3449445837893738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 39/40: 100%|██████████| 416/416 [00:03<00:00, 118.05it/s, loss=0.343]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34293622971297455\n",
      "{'roc_auc': 0.6781034825805736}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 40/40: 100%|██████████| 416/416 [00:03<00:00, 106.01it/s, loss=0.348]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3426130780090506\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.69864 (at 9)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.69062 (at 5)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/40: 100%|██████████| 416/416 [00:03<00:00, 116.70it/s, loss=0.399]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39468963938550305\n",
      "{'roc_auc': 0.5724367463297051}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/40: 100%|██████████| 416/416 [00:03<00:00, 120.52it/s, loss=0.379]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3703582728496538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/40: 100%|██████████| 416/416 [00:03<00:00, 110.89it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36523613039977276\n",
      "{'roc_auc': 0.6419974362570657}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/40: 100%|██████████| 416/416 [00:03<00:00, 121.36it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36510205978097826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/40: 100%|██████████| 416/416 [00:03<00:00, 114.95it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3639529809140815\n",
      "{'roc_auc': 0.6668887381231183}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/40: 100%|██████████| 416/416 [00:03<00:00, 110.65it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36233696757027734\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/40: 100%|██████████| 416/416 [00:03<00:00, 114.09it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3620369224092708\n",
      "{'roc_auc': 0.682689516378925}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/40: 100%|██████████| 416/416 [00:03<00:00, 118.48it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36207010296101755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/40: 100%|██████████| 416/416 [00:03<00:00, 114.41it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36052398996141094\n",
      "{'roc_auc': 0.6823535551547216}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/40: 100%|██████████| 416/416 [00:03<00:00, 119.21it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36024586293989647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/40: 100%|██████████| 416/416 [00:03<00:00, 118.21it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36015138291538906\n",
      "{'roc_auc': 0.6867748701004296}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/40: 100%|██████████| 416/416 [00:03<00:00, 111.83it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3609957517697834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/40: 100%|██████████| 416/416 [00:03<00:00, 111.85it/s, loss=0.371]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3591063803849885\n",
      "{'roc_auc': 0.682625096629624}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/40: 100%|██████████| 416/416 [00:03<00:00, 118.49it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35997471790044355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/40: 100%|██████████| 416/416 [00:03<00:00, 114.88it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35923252587851423\n",
      "{'roc_auc': 0.6856854424413616}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/40: 100%|██████████| 416/416 [00:03<00:00, 120.00it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35858225618274164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/40: 100%|██████████| 416/416 [00:03<00:00, 118.34it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35833024169103456\n",
      "{'roc_auc': 0.6824041124263249}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/40: 100%|██████████| 416/416 [00:03<00:00, 114.13it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3577275124665063\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/40: 100%|██████████| 416/416 [00:03<00:00, 112.13it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3625009669564091\n",
      "{'roc_auc': 0.6865334999005164}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/40: 100%|██████████| 416/416 [00:03<00:00, 117.76it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35827025258913636\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/40: 100%|██████████| 416/416 [00:03<00:00, 115.94it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3575618965551257\n",
      "{'roc_auc': 0.6806770107931621}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/40: 100%|██████████| 416/416 [00:03<00:00, 114.72it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35787842236459255\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/40: 100%|██████████| 416/416 [00:03<00:00, 114.04it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35653543391694814\n",
      "{'roc_auc': 0.6876049878825635}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/40: 100%|██████████| 416/416 [00:03<00:00, 113.72it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35666066966950893\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/40: 100%|██████████| 416/416 [00:03<00:00, 107.06it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35653754241334706\n",
      "{'roc_auc': 0.6903244798309104}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/40: 100%|██████████| 416/416 [00:03<00:00, 118.81it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3570920842198225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/40: 100%|██████████| 416/416 [00:03<00:00, 117.73it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3555891584031857\n",
      "{'roc_auc': 0.6856454858879977}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/40: 100%|██████████| 416/416 [00:03<00:00, 114.17it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.355603308775104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/40: 100%|██████████| 416/416 [00:03<00:00, 118.29it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3546346676750825\n",
      "{'roc_auc': 0.686023849985159}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/40: 100%|██████████| 416/416 [00:03<00:00, 114.28it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35418117705446023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 31/40: 100%|██████████| 416/416 [00:03<00:00, 108.53it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3536785119929566\n",
      "{'roc_auc': 0.6808955486768673}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 32/40: 100%|██████████| 416/416 [00:03<00:00, 119.83it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35277601940414083\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 33/40: 100%|██████████| 416/416 [00:03<00:00, 120.27it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3523934569854576\n",
      "{'roc_auc': 0.6848716334565191}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 34/40: 100%|██████████| 416/416 [00:03<00:00, 110.26it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35331382405442685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 35/40: 100%|██████████| 416/416 [00:03<00:00, 116.23it/s, loss=0.355]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35126916258237684\n",
      "{'roc_auc': 0.6749970644164875}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 36/40: 100%|██████████| 416/416 [00:03<00:00, 115.40it/s, loss=0.354]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35103680616101396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 37/40: 100%|██████████| 416/416 [00:03<00:00, 111.24it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3501825934061064\n",
      "{'roc_auc': 0.6772529788018252}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 38/40: 100%|██████████| 416/416 [00:03<00:00, 116.74it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34976397619511074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 39/40: 100%|██████████| 416/416 [00:03<00:00, 117.60it/s, loss=0.349]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3481724185224336\n",
      "{'roc_auc': 0.6797490402272793}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 40/40: 100%|██████████| 416/416 [00:03<00:00, 116.06it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3480253074891292\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.69864 (at 9)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.69062 (at 5)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.001\n",
      "roc_auc: 0.69032 (at 12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/40: 100%|██████████| 416/416 [00:03<00:00, 106.55it/s, loss=0.393]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.38982709067372173\n",
      "{'roc_auc': 0.6185103218378057}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/40: 100%|██████████| 416/416 [00:03<00:00, 116.75it/s, loss=0.373]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3687841926629727\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/40: 100%|██████████| 416/416 [00:03<00:00, 104.38it/s, loss=0.375]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36826589932808507\n",
      "{'roc_auc': 0.643090125675592}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/40: 100%|██████████| 416/416 [00:03<00:00, 111.00it/s, loss=0.373]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36495059959662074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/40: 100%|██████████| 416/416 [00:03<00:00, 107.53it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3641633437230037\n",
      "{'roc_auc': 0.6599191736006237}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/40: 100%|██████████| 416/416 [00:03<00:00, 105.81it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3621184014213773\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/40: 100%|██████████| 416/416 [00:03<00:00, 106.75it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3628242314220048\n",
      "{'roc_auc': 0.6568319182733551}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/40: 100%|██████████| 416/416 [00:03<00:00, 111.26it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3623768165349387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/40: 100%|██████████| 416/416 [00:03<00:00, 105.21it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36206314268593603\n",
      "{'roc_auc': 0.6681094516003823}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/40: 100%|██████████| 416/416 [00:03<00:00, 113.86it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3614378795744135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/40: 100%|██████████| 416/416 [00:03<00:00, 107.99it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36115670376099074\n",
      "{'roc_auc': 0.6773198448707201}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/40: 100%|██████████| 416/416 [00:03<00:00, 105.28it/s, loss=0.371]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36107305659411043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/40: 100%|██████████| 416/416 [00:03<00:00, 108.64it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3611259121232881\n",
      "{'roc_auc': 0.6796161235293543}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/40: 100%|██████████| 416/416 [00:03<00:00, 109.57it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3597952211633898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/40: 100%|██████████| 416/416 [00:04<00:00, 99.13it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3594162012450397\n",
      "{'roc_auc': 0.6890784877178449}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/40: 100%|██████████| 416/416 [00:03<00:00, 112.25it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36025371531454414\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/40: 100%|██████████| 416/416 [00:03<00:00, 108.52it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3593438217201485\n",
      "{'roc_auc': 0.6813032686091531}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/40: 100%|██████████| 416/416 [00:03<00:00, 109.34it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35969387439007944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/40: 100%|██████████| 416/416 [00:03<00:00, 106.15it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35875417317192143\n",
      "{'roc_auc': 0.6753269098417068}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/40: 100%|██████████| 416/416 [00:04<00:00, 103.32it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.358602231905724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/40: 100%|██████████| 416/416 [00:03<00:00, 105.46it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3592077687454338\n",
      "{'roc_auc': 0.6738599335253422}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/40: 100%|██████████| 416/416 [00:03<00:00, 109.91it/s, loss=0.369]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3588670352832056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/40: 100%|██████████| 416/416 [00:03<00:00, 107.81it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35859854698467714\n",
      "{'roc_auc': 0.6652081165622359}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/40: 100%|██████████| 416/416 [00:03<00:00, 111.01it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35724735582390654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/40: 100%|██████████| 416/416 [00:03<00:00, 106.41it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35756412495930606\n",
      "{'roc_auc': 0.6717177730011122}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/40: 100%|██████████| 416/416 [00:04<00:00, 100.47it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35627225446156585\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/40: 100%|██████████| 416/416 [00:03<00:00, 106.97it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35770912640369856\n",
      "{'roc_auc': 0.6767139730513433}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/40: 100%|██████████| 416/416 [00:03<00:00, 107.96it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3561399055358309\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/40: 100%|██████████| 416/416 [00:04<00:00, 103.09it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35509825956362945\n",
      "{'roc_auc': 0.681141811515968}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/40: 100%|██████████| 416/416 [00:03<00:00, 113.21it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3547489572531329\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 31/40: 100%|██████████| 416/416 [00:03<00:00, 108.16it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3550462440157739\n",
      "{'roc_auc': 0.6586866362453234}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 32/40: 100%|██████████| 416/416 [00:03<00:00, 105.87it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35484979222886837\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 33/40: 100%|██████████| 416/416 [00:03<00:00, 109.37it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35364616952406674\n",
      "{'roc_auc': 0.6757688782483047}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 34/40: 100%|██████████| 416/416 [00:03<00:00, 104.49it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35345929194814885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 35/40: 100%|██████████| 416/416 [00:03<00:00, 104.27it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3534962309237856\n",
      "{'roc_auc': 0.6746863818280857}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 36/40: 100%|██████████| 416/416 [00:03<00:00, 110.25it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35228341069215763\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 37/40: 100%|██████████| 416/416 [00:03<00:00, 106.00it/s, loss=0.353]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35081810280322456\n",
      "{'roc_auc': 0.6687854512481123}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 38/40: 100%|██████████| 416/416 [00:03<00:00, 111.32it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3514937419229402\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 39/40: 100%|██████████| 416/416 [00:03<00:00, 108.30it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.351033529624916\n",
      "{'roc_auc': 0.6608027026938872}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 40/40: 100%|██████████| 416/416 [00:03<00:00, 111.92it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3502346588513599\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.69864 (at 9)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.69062 (at 5)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.001\n",
      "roc_auc: 0.69032 (at 12)\n",
      "lr=0.001, hidden=128, nl=3, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.68908 (at 7)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/40: 100%|██████████| 416/416 [00:03<00:00, 105.24it/s, loss=0.397]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39178701801798665\n",
      "{'roc_auc': 0.5771275641506541}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/40: 100%|██████████| 416/416 [00:03<00:00, 110.59it/s, loss=0.379]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.370042606901664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/40: 100%|██████████| 416/416 [00:03<00:00, 105.15it/s, loss=0.375]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3673086941170578\n",
      "{'roc_auc': 0.6376462491397109}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/40: 100%|██████████| 416/416 [00:03<00:00, 110.58it/s, loss=0.373]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3660018094098912\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/40: 100%|██████████| 416/416 [00:03<00:00, 107.92it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3642632997451493\n",
      "{'roc_auc': 0.652831370297766}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/40: 100%|██████████| 416/416 [00:03<00:00, 108.86it/s, loss=0.371]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36265970358195215\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/40: 100%|██████████| 416/416 [00:03<00:00, 108.87it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3619167504545588\n",
      "{'roc_auc': 0.6819009860298842}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/40: 100%|██████████| 416/416 [00:03<00:00, 112.23it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36176895060075015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/40: 100%|██████████| 416/416 [00:03<00:00, 107.43it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36227712977247745\n",
      "{'roc_auc': 0.6807210445458489}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/40: 100%|██████████| 416/416 [00:03<00:00, 116.07it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36114625783207327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/40: 100%|██████████| 416/416 [00:03<00:00, 111.36it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3597259815925589\n",
      "{'roc_auc': 0.6788422710978756}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/40: 100%|██████████| 416/416 [00:03<00:00, 106.78it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3608039159041185\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/40: 100%|██████████| 416/416 [00:03<00:00, 108.55it/s, loss=0.371]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36071993826100457\n",
      "{'roc_auc': 0.6755837733990468}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/40: 100%|██████████| 416/416 [00:03<00:00, 108.95it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3604384086524638\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/40: 100%|██████████| 416/416 [00:03<00:00, 107.10it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3600783826281818\n",
      "{'roc_auc': 0.6807646705786036}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/40: 100%|██████████| 416/416 [00:03<00:00, 114.01it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35904719290108633\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/40: 100%|██████████| 416/416 [00:03<00:00, 111.91it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3597744628070639\n",
      "{'roc_auc': 0.6643388576666025}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/40: 100%|██████████| 416/416 [00:03<00:00, 116.24it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3576845498397373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/40: 100%|██████████| 416/416 [00:03<00:00, 110.97it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3579494279021254\n",
      "{'roc_auc': 0.6728072006601801}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/40: 100%|██████████| 416/416 [00:03<00:00, 112.95it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3591708813865597\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/40: 100%|██████████| 416/416 [00:03<00:00, 111.70it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3584564125451904\n",
      "{'roc_auc': 0.6695095618478519}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/40: 100%|██████████| 416/416 [00:03<00:00, 112.91it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3584832069822229\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/40: 100%|██████████| 416/416 [00:03<00:00, 109.10it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3578549209767236\n",
      "{'roc_auc': 0.6800686926541916}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/40: 100%|██████████| 416/416 [00:03<00:00, 116.15it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35734937279126966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/40: 100%|██████████| 416/416 [00:03<00:00, 109.14it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3569111946540383\n",
      "{'roc_auc': 0.6719326414054269}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/40: 100%|██████████| 416/416 [00:03<00:00, 117.33it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35629536548199564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/40: 100%|██████████| 416/416 [00:03<00:00, 113.81it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35517391193514836\n",
      "{'roc_auc': 0.6647119214046442}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/40: 100%|██████████| 416/416 [00:03<00:00, 118.85it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3562752189687811\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/40: 100%|██████████| 416/416 [00:03<00:00, 112.31it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35645741607564\n",
      "{'roc_auc': 0.6725748002987773}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/40: 100%|██████████| 416/416 [00:03<00:00, 108.72it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3550710128620267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 31/40: 100%|██████████| 416/416 [00:03<00:00, 109.35it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3538264657657307\n",
      "{'roc_auc': 0.6637550027235691}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 32/40: 100%|██████████| 416/416 [00:03<00:00, 107.79it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3535194080322981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 33/40: 100%|██████████| 416/416 [00:03<00:00, 111.05it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3532288156163234\n",
      "{'roc_auc': 0.6764726028514301}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 34/40: 100%|██████████| 416/416 [00:03<00:00, 115.47it/s, loss=0.355]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3522057962030746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 35/40: 100%|██████████| 416/416 [00:03<00:00, 107.68it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.353086305459818\n",
      "{'roc_auc': 0.6693395426360889}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 36/40: 100%|██████████| 416/416 [00:03<00:00, 117.39it/s, loss=0.36]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3511432670773222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 37/40: 100%|██████████| 416/416 [00:03<00:00, 107.96it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3520211905527573\n",
      "{'roc_auc': 0.6705223381596501}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 38/40: 100%|██████████| 416/416 [00:03<00:00, 117.17it/s, loss=0.355]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3502546463591548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 39/40: 100%|██████████| 416/416 [00:03<00:00, 110.20it/s, loss=0.355]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34963189939466804\n",
      "{'roc_auc': 0.6673771866019969}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 40/40: 100%|██████████| 416/416 [00:03<00:00, 115.90it/s, loss=0.356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34741672235899246\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.69864 (at 9)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.69062 (at 5)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.001\n",
      "roc_auc: 0.69032 (at 12)\n",
      "lr=0.001, hidden=128, nl=3, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.68908 (at 7)\n",
      "lr=0.001, hidden=128, nl=3, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.6819 (at 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/40: 100%|██████████| 416/416 [00:03<00:00, 110.11it/s, loss=0.404]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3931394934725876\n",
      "{'roc_auc': 0.6008372936529423}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/40: 100%|██████████| 416/416 [00:03<00:00, 112.40it/s, loss=0.38]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37041098757002217\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/40: 100%|██████████| 416/416 [00:03<00:00, 110.89it/s, loss=0.374]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3665484352252231\n",
      "{'roc_auc': 0.6567328423298096}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/40: 100%|██████████| 416/416 [00:03<00:00, 109.29it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3660539271166691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/40: 100%|██████████| 416/416 [00:03<00:00, 106.54it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.363893526355521\n",
      "{'roc_auc': 0.6590401294266153}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/40: 100%|██████████| 416/416 [00:03<00:00, 114.72it/s, loss=0.372]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36436239518941593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/40: 100%|██████████| 416/416 [00:03<00:00, 108.19it/s, loss=0.372]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.362414280561587\n",
      "{'roc_auc': 0.67195995864089}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/40: 100%|██████████| 416/416 [00:03<00:00, 115.91it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3630519116369004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/40: 100%|██████████| 416/416 [00:03<00:00, 110.91it/s, loss=0.368]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3613425996512748\n",
      "{'roc_auc': 0.6778645587002541}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/40: 100%|██████████| 416/416 [00:03<00:00, 113.12it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3621252879070548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/40: 100%|██████████| 416/416 [00:03<00:00, 111.73it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36152388635449684\n",
      "{'roc_auc': 0.6874084668752019}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/40: 100%|██████████| 416/416 [00:03<00:00, 110.19it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3603097188214843\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/40: 100%|██████████| 416/416 [00:03<00:00, 110.39it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.36095595015929294\n",
      "{'roc_auc': 0.6857710636271418}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/40: 100%|██████████| 416/416 [00:03<00:00, 113.33it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3597225519613578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/40: 100%|██████████| 416/416 [00:03<00:00, 114.99it/s, loss=0.37]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3606788679026067\n",
      "{'roc_auc': 0.6811434423956971}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/40: 100%|██████████| 416/416 [00:03<00:00, 115.79it/s, loss=0.363]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.359930979875991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/40: 100%|██████████| 416/416 [00:03<00:00, 109.38it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3600090860317533\n",
      "{'roc_auc': 0.6941521545552102}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/40: 100%|██████████| 416/416 [00:03<00:00, 114.96it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35879998919195855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/40: 100%|██████████| 416/416 [00:03<00:00, 110.41it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35798987989815384\n",
      "{'roc_auc': 0.6946214401972712}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/40: 100%|██████████| 416/416 [00:03<00:00, 113.96it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3591798092955007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/40: 100%|██████████| 416/416 [00:03<00:00, 111.76it/s, loss=0.367]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35872897799485004\n",
      "{'roc_auc': 0.6925649008588213}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/40: 100%|██████████| 416/416 [00:03<00:00, 112.13it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.357792466878891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/40: 100%|██████████| 416/416 [00:03<00:00, 108.01it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3568300736638216\n",
      "{'roc_auc': 0.6864095530411014}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/40: 100%|██████████| 416/416 [00:03<00:00, 108.00it/s, loss=0.365]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3562572594338025\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/40: 100%|██████████| 416/416 [00:03<00:00, 106.15it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3571192070913429\n",
      "{'roc_auc': 0.6800389290991347}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/40: 100%|██████████| 416/416 [00:03<00:00, 110.82it/s, loss=0.366]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3558407780141212\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/40: 100%|██████████| 416/416 [00:03<00:00, 111.64it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35584940073581844\n",
      "{'roc_auc': 0.6901907476931206}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/40: 100%|██████████| 416/416 [00:03<00:00, 109.96it/s, loss=0.359]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3560719278712685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/40: 100%|██████████| 416/416 [00:03<00:00, 108.98it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35561849126735556\n",
      "{'roc_auc': 0.6830491253592013}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/40: 100%|██████████| 416/416 [00:03<00:00, 109.37it/s, loss=0.364]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3555788997059258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 31/40: 100%|██████████| 416/416 [00:04<00:00, 102.31it/s, loss=0.362]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35405828803777695\n",
      "{'roc_auc': 0.6857710636271417}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 32/40: 100%|██████████| 416/416 [00:03<00:00, 104.48it/s, loss=0.361]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3536813332197758\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 33/40: 100%|██████████| 416/416 [00:03<00:00, 111.74it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35314215873726285\n",
      "{'roc_auc': 0.6851268661341301}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 34/40: 100%|██████████| 416/416 [00:03<00:00, 110.48it/s, loss=0.354]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3527523840849216\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 35/40: 100%|██████████| 416/416 [00:03<00:00, 108.86it/s, loss=0.358]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3517169152171566\n",
      "{'roc_auc': 0.6807740481370461}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 36/40: 100%|██████████| 416/416 [00:03<00:00, 108.58it/s, loss=0.354]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35173339772826206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 37/40: 100%|██████████| 416/416 [00:03<00:00, 113.96it/s, loss=0.355]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3520322762644635\n",
      "{'roc_auc': 0.6808776089998467}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 38/40: 100%|██████████| 416/416 [00:03<00:00, 104.09it/s, loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35038532119674176\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 39/40: 100%|██████████| 416/416 [00:04<00:00, 101.54it/s, loss=0.354]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3494231758209375\n",
      "{'roc_auc': 0.6744446039082402}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 40/40: 100%|██████████| 416/416 [00:03<00:00, 105.94it/s, loss=0.348]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3474076883509182\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.69864 (at 9)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.69062 (at 5)\n",
      "lr=0.001, hidden=128, nl=2, dropout=0.5, decay=0.001\n",
      "roc_auc: 0.69032 (at 12)\n",
      "lr=0.001, hidden=128, nl=3, dropout=0.5, decay=0.1\n",
      "roc_auc: 0.68908 (at 7)\n",
      "lr=0.001, hidden=128, nl=3, dropout=0.5, decay=0.01\n",
      "roc_auc: 0.6819 (at 3)\n",
      "lr=0.001, hidden=128, nl=3, dropout=0.5, decay=0.001\n",
      "roc_auc: 0.69462 (at 9)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "  # Set up logging\n",
    "from itertools import product\n",
    "\n",
    "# Define the grid of hyperparameters\n",
    "grid = {\n",
    "    'lr': [0.001],\n",
    "    'hidden_size': [128],\n",
    "    'num_layers': [1, 2, 3],\n",
    "    'dropout': [0.5],\n",
    "    'weight_decay': [0.1, 0.01, 0.001],\n",
    "}\n",
    "\n",
    "all_results = {}\n",
    "\n",
    "for hyperparams in product(*grid.values()):\n",
    "    lr, hidden_size, num_layers, dropout, weight_decay = hyperparams\n",
    "    if num_layers == 1 and dropout:\n",
    "      continue\n",
    "    desc = f\"lr={lr}, hidden={hidden_size}, nl={num_layers}, dropout={dropout}, decay={weight_decay}\"\n",
    "\n",
    "    results = train(epochs=40, hidden_size=hidden_size, num_layers=num_layers, lr=lr, dropout=dropout)\n",
    "    all_results[desc] = results\n",
    "\n",
    "    print_max_values(all_results)\n",
    "\n",
    "    # Update the best hyperparameters and the best score if the current score is higher\n",
    "    #print(score)\n",
    "    #if score > best_score:\n",
    "     #   best_hyperparams = hyperparams\n",
    "       # best_score = score\n",
    "\n",
    "# Print the best hyperparameters and the best score\n",
    "#print(f\"Best Hyperparameters: {best_hyperparams}\")\n",
    "#print(f\"Best ROC AUC Score: {best_score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2QwRscyAk_mT",
   "metadata": {
    "id": "2QwRscyAk_mT"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}