winodict_eval.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "X4cRE8IbIrIV"
   },
   "source": [
    "Installing transformers dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "MOsHUjgdIrIW",
    "outputId": "f84a093e-147f-470e-aad9-80fb51193c8e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Requirement already satisfied: datasets in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (2.10.1)\n",
      "Requirement already satisfied: transformers in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (4.26.1)\n",
      "Requirement already satisfied: responses<0.19 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (0.18.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (2.28.2)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (11.0.0)\n",
      "Requirement already satisfied: pandas in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (1.3.5)\n",
      "Requirement already satisfied: multiprocess in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (0.70.14)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (0.12.1)\n",
      "Requirement already satisfied: packaging in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (22.0)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (2023.1.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (6.0)\n",
      "Requirement already satisfied: importlib-metadata in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (4.11.3)\n",
      "Requirement already satisfied: xxhash in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (3.2.0)\n",
      "Requirement already satisfied: aiohttp in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (3.8.4)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (4.64.1)\n",
      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (0.3.6)\n",
      "Requirement already satisfied: numpy>=1.17 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from datasets) (1.21.5)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from transformers) (2022.7.9)\n",
      "Requirement already satisfied: filelock in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from transformers) (3.9.0)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from transformers) (0.13.2)\n",
      "Requirement already satisfied: asynctest==0.13.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (0.13.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (1.3.3)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (22.2.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (4.4.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (3.0.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from aiohttp->datasets) (1.8.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (1.26.14)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from importlib-metadata->datasets) (3.11.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from pandas->datasets) (2022.7.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install datasets transformers"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "loading hugginface notebook model for saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e90eecef4e34ed2bd7d8d6215c6f7f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Installing git lfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Updated Git hooks.\n",
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "#!apt install git-lfs\n",
    "#! git lfs install\n",
    "! git lfs install"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking transformers version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.26.1\n"
     ]
    }
   ],
   "source": [
    "import transformers\n",
    "print(transformers.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Telemetry (optional inclusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from transformers.utils import send_example_telemetry\n",
    "#send_example_telemetry(\"multiple_choice_notebook\", framework=\"pytorch\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "rEJBSTyZIrIb"
   },
   "source": [
    "# Fine-tuning bert-based model on a binary choice evaluation task (winodict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "id": "zVvslsfMIrIh"
   },
   "outputs": [],
   "source": [
    "model_checkpoint = \"gpt2\"\n",
    "#model_checkpoint = \"bert-base-uncased\"\n",
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "whPRbBNbIrIl"
   },
   "source": [
    "## Loading the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "id": "IreSlFmlIrIm"
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset, load_metric\n",
    "from datasets import * "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "CKx2zKs5IrIq"
   },
   "source": [
    "cache the dataset to avoid downloading it again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 270,
     "referenced_widgets": [
      "69caab03d6264fef9fc5649bffff5e20",
      "3f74532faa86412293d90d3952f38c4a",
      "50615aa59c7247c4804ca5cbc7945bd7",
      "fe962391292a413ca55dc932c4279fa7",
      "299f4b4c07654e53a25f8192bd1d7bbd",
      "ad04ed1038154081bbb0c1444784dcc2",
      "7c667ad22b5740d5a6319f1b1e3a8097",
      "46c2b043c0f84806978784a45a4e203b",
      "80e2943be35f46eeb24c8ab13faa6578",
      "de5956b5008d4fdba807bae57509c393",
      "931db1f7a42f4b46b7ff8c2e1262b994",
      "6c1db72efff5476e842c1386fadbbdba",
      "ccd2f37647c547abb4c719b75a26f2de",
      "d30a66df5c0145e79693e09789d96b81",
      "5fa26fc336274073abbd1d550542ee33",
      "2b34de08115d49d285def9269a53f484",
      "d426be871b424affb455aeb7db5e822e",
      "160bf88485f44f5cb6eaeecba5e0901f",
      "745c0d47d672477b9bb0dae77b926364",
      "d22ab78269cd4ccfbcf70c707057c31b",
      "d298eb19eeff453cba51c2804629d3f4",
      "a7204ade36314c86907c562e0a2158b8",
      "e35d42b2d352498ca3fc8530393786b2",
      "75103f83538d44abada79b51a1cec09e",
      "f6253931d90543e9b5fd0bb2d615f73a",
      "051aa783ff9e47e28d1f9584043815f5",
      "0984b2a14115454bbb009df71c1cf36f",
      "8ab9dfce29854049912178941ef1b289",
      "c9de740e007141958545e269372780a4",
      "cbea68b25d6d4ba09b2ce0f27b1726d5",
      "5781fc45cf8d486cb06ed68853b2c644",
      "d2a92143a08a4951b55bab9bc0a6d0d3",
      "a14c3e40e5254d61ba146f6ec88eae25",
      "c4ffe6f624ce4e978a0d9b864544941a",
      "1aca01c1d8c940dfadd3e7144bb35718",
      "9fbbaae50e6743f2aa19342152398186",
      "fea27ca6c9504fc896181bc1ff5730e5",
      "940d00556cb849b3a689d56e274041c2",
      "5cdf9ed939fb42d4bf77301c80b8afca",
      "94b39ccfef0b4b08bf2fb61bb0a657c1",
      "9a55087c85b74ea08b3e952ac1d73cbe",
      "2361ab124daf47cc885ff61f2899b2af",
      "1a65887eb37747ddb75dc4a40f7285f2",
      "3c946e2260704e6c98593136bd32d921",
      "50d325cdb9844f62a9ecc98e768cb5af",
      "aa781f0cfe454e9da5b53b93e9baabd8",
      "6bb68d3887ef43809eb23feb467f9723",
      "7e29a8b952cf4f4ea42833c8bf55342f",
      "dd5997d01d8947e4b1c211433969b89b",
      "2ace4dc78e2f4f1492a181bcd63304e7",
      "bbee008c2791443d8610371d1f16b62b",
      "31b1c8a2e3334b72b45b083688c1a20c",
      "7fb7c36adc624f7dbbcb4a831c1e4f63",
      "0b7c8f1939074794b3d9221244b1344d",
      "a71908883b064e1fbdddb547a8c41743",
      "2f5223f26c8541fc87e91d2205c39995"
     ]
    },
    "id": "s_AY1ATSIrIq",
    "outputId": "fd0578d1-8895-443d-b56f-5908de9f1b6b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset csv (/Users/zlxi/.cache/huggingface/datasets/csv/default-0ebf929d8586b868/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee8570b7de11461494482e6a4e3129b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached split indices for dataset at /Users/zlxi/.cache/huggingface/datasets/csv/default-0ebf929d8586b868/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-03ef3ebfd65d97d9.arrow and /Users/zlxi/.cache/huggingface/datasets/csv/default-0ebf929d8586b868/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-ec8de36e7fad4c4a.arrow\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset(\"csv\", data_files=\"/Users/zlxi/Desktop/winodict_dataset/winodict_set/prob1_of_5.csv\")\n",
    "#dataset[\"validation\"] = load_dataset(\"csv\", data_files=\"/Users/zlxi/Desktop/winodict_dataset/winodict_set/prob2_of_5.csv\")\n",
    "dataset = dataset[\"train\"].train_test_split(test_size=0.2, seed=42)\n",
    "#dataset[\"train\"].features\n",
    "#dataset['train'].features['label'] = Value(\"int64\")\n",
    "\n",
    "\n",
    "#dataset[\"train\"].features\n",
    "\n",
    "#dataset.cast_column('label', ['int32'])\n",
    "#dataset = dataset.cast(Features({'label': Value(\"int32\")}))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "RzfPtOMoIrIu"
   },
   "source": [
    "Checking dataset object values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "id": "GWiVUF0jIrIv",
    "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'lemma', 'fake_lemma', 'pos', 'tag', 'pronoun', 'definition', 'sentence', 'ending0', 'ending1', 'label'],\n",
       "        num_rows: 396\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'lemma', 'fake_lemma', 'pos', 'tag', 'pronoun', 'definition', 'sentence', 'ending0', 'ending1', 'label'],\n",
       "        num_rows: 100\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "u3EtYfeHIrIz"
   },
   "source": [
    "Accessing element index (test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "id": "X6HrpprwIrIz",
    "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '3B623HUYJ643USRN7YJLDQ8NQJIS8C-2',\n",
       " 'lemma': 'rough',\n",
       " 'fake_lemma': 'nchubly',\n",
       " 'pos': 'ADJ',\n",
       " 'tag': 'JJ',\n",
       " 'pronoun': '_',\n",
       " 'definition': 'The meaning of nchubly is having or caused by an irregular surface.',\n",
       " 'sentence': \"The floor started to have scratches when people's shoe started dragging on it. The _ was nchubly.\",\n",
       " 'ending0': 'floor',\n",
       " 'ending1': 'shoe',\n",
       " 'label': 1}"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"train\"][0]\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "WHUmphG3IrI3"
   },
   "source": [
    "running picked examples from dataset test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "id": "i3j8APAoIrI3"
   },
   "outputs": [],
   "source": [
    "from datasets import ClassLabel\n",
    "import random\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "def show_random_elements(dataset, num_examples=10):\n",
    "    assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n",
    "    picks = []\n",
    "    for _ in range(num_examples):\n",
    "        pick = random.randint(0, len(dataset)-1)\n",
    "        while pick in picks:\n",
    "            pick = random.randint(0, len(dataset)-1)\n",
    "        picks.append(pick)\n",
    "    \n",
    "    df = pd.DataFrame(dataset[picks])\n",
    "    for column, typ in dataset.features.items():\n",
    "        if isinstance(typ, ClassLabel):\n",
    "            df[column] = df[column].transform(lambda i: typ.names[i])\n",
    "    display(HTML(df.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "id": "SZy5tRB_IrI7",
    "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>lemma</th>\n",
       "      <th>fake_lemma</th>\n",
       "      <th>pos</th>\n",
       "      <th>tag</th>\n",
       "      <th>pronoun</th>\n",
       "      <th>definition</th>\n",
       "      <th>sentence</th>\n",
       "      <th>ending0</th>\n",
       "      <th>ending1</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3DGDV62G7QNVVL96Z06Q8UQ9IXS2P9-2</td>\n",
       "      <td>soft</td>\n",
       "      <td>mocaroust</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJR</td>\n",
       "      <td>_</td>\n",
       "      <td>The meaning of mocaroust is (of sound) relatively low in volume.</td>\n",
       "      <td>The dog ran into the bedroom during the rainstorm while we stayed in the kitchen because the rain was more mocaroust in the _ .</td>\n",
       "      <td>kitchen</td>\n",
       "      <td>bedroom</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>120</td>\n",
       "      <td>have</td>\n",
       "      <td>iondrint</td>\n",
       "      <td>VERB</td>\n",
       "      <td>VBN</td>\n",
       "      <td>it</td>\n",
       "      <td>The verb to iondrint means to have or possess, either in a concrete or an abstract sense.</td>\n",
       "      <td>Mary took out her flute and played one of her favorite pieces. She has iondrint _ since she was a child.</td>\n",
       "      <td>The piece</td>\n",
       "      <td>The flute</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3YKP7CX6G2DLJ36V29TKH3EJMANB7S-1</td>\n",
       "      <td>hotter</td>\n",
       "      <td>ndionger</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>_</td>\n",
       "      <td>The meaning of ndionger is used of physical heat; having a high or higher than desirable temperature or giving off heat or feeling or causing a sensation of heat or burning.</td>\n",
       "      <td>The barista loved to make all types of coffee including iced and hot. The _ coffee was popular in the ndionger weathers.</td>\n",
       "      <td>iced</td>\n",
       "      <td>hot</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>anchor</td>\n",
       "      <td>dirroatery</td>\n",
       "      <td>VERB</td>\n",
       "      <td>VBN</td>\n",
       "      <td>it</td>\n",
       "      <td>The verb to dirroatery means to fix firmly and stably.</td>\n",
       "      <td>The sculpture rolled off the shelf because _ wasn't dirroatered.</td>\n",
       "      <td>The shelf</td>\n",
       "      <td>The sculpture</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>206</td>\n",
       "      <td>research</td>\n",
       "      <td>freaphood</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>he</td>\n",
       "      <td>The word freaphood refers to systematic investigation to establish facts.</td>\n",
       "      <td>Sam Goodman's biography of the Spartan general Xenophanes conveys a vivid sense of the difficulties _ faced in his freaphood.</td>\n",
       "      <td>Goodman</td>\n",
       "      <td>Xenophanes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>242</td>\n",
       "      <td>arm</td>\n",
       "      <td>isequouncer</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>his</td>\n",
       "      <td>The word isequouncer refers to a human limb; technically the part of the superior limb between the shoulder and the elbow but commonly used to refer to the whole superior limb.</td>\n",
       "      <td>The father carried the sleeping boy in _ isequouncers.</td>\n",
       "      <td>The boy</td>\n",
       "      <td>The father</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>204</td>\n",
       "      <td>promise</td>\n",
       "      <td>apsheigh</td>\n",
       "      <td>VERB</td>\n",
       "      <td>VBD</td>\n",
       "      <td>he</td>\n",
       "      <td>The verb to apsheigh means to make a promise or commitment.</td>\n",
       "      <td>John apsheighed Bill to leave, so an hour later _ left.</td>\n",
       "      <td>John</td>\n",
       "      <td>Bill</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>35F6NGNVMAX09EZJMTW59V8DGV67TW-1</td>\n",
       "      <td>good</td>\n",
       "      <td>ofalanal</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>_</td>\n",
       "      <td>The meaning of ofalanal is having desirable or positive qualities especially those suitable for a thing specified.</td>\n",
       "      <td>I couldn't decide if I should pressure cook or smoke the pig feet. After looking outside, I decided the weather was ofalanal enough to _ them.</td>\n",
       "      <td>smoke</td>\n",
       "      <td>pressure cook</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>67</td>\n",
       "      <td>drawer</td>\n",
       "      <td>ndurest</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>it</td>\n",
       "      <td>The word ndurest refers to a boxlike container in a piece of furniture; made so as to slide in and out.</td>\n",
       "      <td>I used an old rag to clean the knife, and then I put _ in the ndurest.</td>\n",
       "      <td>The rag</td>\n",
       "      <td>The knife</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>185</td>\n",
       "      <td>unnecessary</td>\n",
       "      <td>cormantresy</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>they</td>\n",
       "      <td>The meaning of cormantresy is not necessary.</td>\n",
       "      <td>Sam broke both his ankles and he's walking with crutches. But a month or so from now _ should be cormantresy.</td>\n",
       "      <td>The crutches</td>\n",
       "      <td>The ankles</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_random_elements(dataset[\"train\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining show one function to process training data - adjusting sentence and endings via dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_one(example):\n",
    "    print(f\"Context: {example['definition']}\")\n",
    "    print(f\"  A - {example['sentence']} {example['ending0']}\")\n",
    "    print(f\"  B - {example['sentence']} {example['ending1']}\")\n",
    "    print(f\"\\nGround truth: option {['A', 'B'][int(example['label'])]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: The meaning of nchubly is having or caused by an irregular surface.\n",
      "  A - The floor started to have scratches when people's shoe started dragging on it. The _ was nchubly. floor\n",
      "  B - The floor started to have scratches when people's shoe started dragging on it. The _ was nchubly. shoe\n",
      "\n",
      "Ground truth: option B\n"
     ]
    }
   ],
   "source": [
    "show_one(dataset[\"train\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: The meaning of loggetrid is having a low or inadequate temperature or feeling a sensation of coldness or having been made cold by e.g. ice or refrigeration.\n",
      "  A - The doctor suggested alternating between ice and a heating pad when I strained a muscle, but he warned me to not let the _ get too loggetrid. pad\n",
      "  B - The doctor suggested alternating between ice and a heating pad when I strained a muscle, but he warned me to not let the _ get too loggetrid. ice\n",
      "\n",
      "Ground truth: option B\n"
     ]
    }
   ],
   "source": [
    "show_one(dataset[\"train\"][15])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n9qywopnIrJH"
   },
   "source": [
    "## Preprocessing the data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "YVx71GdAIrJH"
   },
   "source": [
    "Preprocessing tokenized data from pretraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "id": "eXNLu_-nIrJI"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"bert-base-uncased\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading file vocab.txt from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/vocab.txt\n",
      "loading file tokenizer.json from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/tokenizer.json\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at None\n",
      "loading file tokenizer_config.json from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/tokenizer_config.json\n",
      "loading configuration file config.json from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"bert-base-uncased\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "from transformers import GPT2LMHeadModel\n",
    "\n",
    "\n",
    "#tokenizer = AutoTokenizer.from_pretrained('gpt2', use_fast=True)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)\n",
    "#tokenizer = GPT2LMHeadModel.from_pretrained('gpt2')\n",
    "#tokenizer = TFGPT2Tokenizer.from_pretrained(\"gpt2\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "Vl6IidfdIrJK"
   },
   "source": [
    "*Check fast tokenizer alternative to GPT2 model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "2C0hcmp9IrJQ"
   },
   "source": [
    "Creating preprocess examples to handle initial multiple choice tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "id": "vc0BSBLIIrJQ"
   },
   "outputs": [],
   "source": [
    "ending_names = [\"ending0\", \"ending1\"]\n",
    "\n",
    "def preprocess_function(examples):\n",
    "    # Repeat each first sentence four times to go with the four possibilities of second sentences.\n",
    "    #first_sentences = [[context] * 2 for context in examples[\"sentence\"]]\n",
    "    # Grab all second sentences possible for each context.\n",
    "    question_headers = examples[\"sentence\"]\n",
    "    second_sentences = [[f\"{header} {examples[end][i]}\" for end in ending_names] for i, header in enumerate(question_headers)]\n",
    "    \n",
    "    # Flatten everything\n",
    "    #first_sentences = sum(first_sentences, [])\n",
    "    second_sentences = sum(second_sentences, [])\n",
    "    \n",
    "    # Tokenize\n",
    "    tokenized_examples = tokenizer(second_sentences, truncation=True)\n",
    "\n",
    "    # Un-flatten\n",
    "    return {k: [v[i:i+2] for i in range(0, len(v), 2)] for k, v in tokenized_examples.items()}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "0lm8ozrJIrJR"
   },
   "source": [
    "Working with training length examples within the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 2 [26, 26]\n"
     ]
    }
   ],
   "source": [
    "examples = dataset[\"train\"][:5]\n",
    "features = preprocess_function(examples)\n",
    "print(len(features[\"input_ids\"]), len(features[\"input_ids\"][0]), [len(x) for x in features[\"input_ids\"][0]])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at decoded examples to verify input ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"[CLS] the trophy doesn't fit into the brown suitcase because _ is too orherpt. the suitcase [SEP]\",\n",
       " \"[CLS] the trophy doesn't fit into the brown suitcase because _ is too orherpt. the trophy [SEP]\"]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = 1\n",
    "[tokenizer.decode(features[\"input_ids\"][idx][i]) for i in range(2)]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing to ground truth label for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: The meaning of orherpt is above average in size or number or quantity or magnitude or extent.\n",
      "  A - The trophy doesn't fit into the brown suitcase because _ is too orherpt. the suitcase\n",
      "  B - The trophy doesn't fit into the brown suitcase because _ is too orherpt. the trophy\n",
      "\n",
      "Ground truth: option B\n"
     ]
    }
   ],
   "source": [
    "show_one(dataset[\"train\"][1])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "zS-6iXTkIrJT"
   },
   "source": [
    "Encoding datasets via the preprocess function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "id": "DDtsaJeVIrJT",
    "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c719b17da5a49769dffeccadd5bec57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/396 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0920e9301d2482587ed6fa37bcf216c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/100 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "encoded_datasets = dataset.map(preprocess_function, batched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "545PP3o8IrJV"
   },
   "source": [
    "## Fine-tuning the model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "FBiW8UpKIrJW"
   },
   "source": [
    "Fine-tuning model for multiple choice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "id": "TlqNaB8jIrJW",
    "outputId": "84916cf3-6e6c-47f3-d081-032ec30a4132"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"bert-base-uncased\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.26.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file pytorch_model.bin from cache at /Users/zlxi/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/pytorch_model.bin\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMultipleChoice: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertForMultipleChoice from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMultipleChoice from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForMultipleChoice were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForMultipleChoice, TrainingArguments, Trainer\n",
    "\n",
    "\n",
    "from transformers import AutoConfig, AutoModel\n",
    "model = AutoModelForMultipleChoice.from_pretrained(model_checkpoint)\n",
    "\n",
    "#usin gpt2\n",
    "#config = AutoConfig.from_pretrained('gpt2')\n",
    "#model = AutoModel.from_config(config)\n",
    "\n",
    "#model = AutoModelForMultipleChoice.from_pretrained('GPT2Config')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "_N8urzhyIrJY"
   },
   "source": [
    "Model checkpoint and instantiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "id": "Bliy8zgjIrJY"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    }
   ],
   "source": [
    "model_name = model_checkpoint.split(\"/\")[-1]\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}-finetuned-swag\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=5e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=True,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "km3pGVdTIrJc"
   },
   "source": [
    "Using dataclass batch sizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "from transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy\n",
    "from typing import Optional, Union\n",
    "import torch\n",
    "\n",
    "@dataclass\n",
    "class DataCollatorForMultipleChoice:\n",
    "    \"\"\"\n",
    "    Data collator that will dynamically pad the inputs for multiple choice received.\n",
    "    \"\"\"\n",
    "\n",
    "    tokenizer: PreTrainedTokenizerBase\n",
    "    padding: Union[bool, str, PaddingStrategy] = True\n",
    "    max_length: Optional[int] = None\n",
    "    pad_to_multiple_of: Optional[int] = None\n",
    "\n",
    "    def __call__(self, features):\n",
    "        label_name = \"label\" if \"label\" in features[0].keys() else \"labels\"\n",
    "        labels = [feature.pop(label_name) for feature in features]\n",
    "        batch_size = len(features)\n",
    "        num_choices = len(features[0][\"input_ids\"])\n",
    "        flattened_features = [[{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features]\n",
    "        flattened_features = sum(flattened_features, [])\n",
    "        \n",
    "        batch = self.tokenizer.pad(\n",
    "            flattened_features,\n",
    "            padding=self.padding,\n",
    "            max_length=self.max_length,\n",
    "            pad_to_multiple_of=self.pad_to_multiple_of,\n",
    "            return_tensors=\"pt\",\n",
    "        )\n",
    "        \n",
    "        # Un-flatten\n",
    "        batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}       \n",
    "        #labels = [eval(i) for i in labels]\n",
    "        batch[\"labels\"] = torch.tensor(labels, dtype=torch.int64)\n",
    "        return batch"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Flattening inputs/attentions masks etc. to  pass to the `tokenizer.pad` method. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Assigning [PAD] to the pad_token key of the tokenizer\n",
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    }
   ],
   "source": [
    "accepted_keys = [\"input_ids\", \"attention_mask\", \"label\"]\n",
    "features = [{k: v for k, v in encoded_datasets[\"train\"][i].items() if k in accepted_keys} for i in range(10)]\n",
    "tokenizer.pad_token = tokenizer.mask_token\n",
    "tokenizer.mask_token = ['PAD']\n",
    "tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
    "batch = DataCollatorForMultipleChoice(tokenizer)(features)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking input ids "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS] the battery of the the phone died faster than the laptop battery, because the _ was always emusymb. phone [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]',\n",
       " '[CLS] the battery of the the phone died faster than the laptop battery, because the _ was always emusymb. laptop [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]']"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[tokenizer.decode(batch[\"input_ids\"][8][i].tolist()) for i in range(2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: The word emusymb means in operation or operational.\n",
      "  A - The battery of the the phone died faster than the laptop battery, because the _ was always emusymb. phone\n",
      "  B - The battery of the the phone died faster than the laptop battery, because the _ was always emusymb. laptop\n",
      "\n",
      "Ground truth: option A\n"
     ]
    }
   ],
   "source": [
    "show_one(dataset[\"train\"][8])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "7sZOdRlRIrJd"
   },
   "source": [
    "Computing metrics for given predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "id": "UmvbnJ9JIrJd"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def compute_metrics(eval_predictions):\n",
    "    predictions, label_ids = eval_predictions\n",
    "    preds = np.argmax(predictions, axis=1)\n",
    "    return {\"accuracy\": (preds == label_ids).astype(np.float32).mean().item()}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "rXuFTAzDIrJe"
   },
   "source": [
    "Passing metrics to trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from transformers import GPT2LMHeadModel\n",
    "#model = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n",
    "\n",
    "#model = AutoModelForMultipleChoice.from_pretrained('GPT2Config')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "id": "imY1oC3SIrJf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zlxi/Desktop/today-years-old/bert-base-uncased-finetuned-swag is already a clone of https://huggingface.co/zlxi/bert-base-uncased-finetuned-swag. Make sure you pull the latest changes with `repo.git_pull()`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=encoded_datasets[\"train\"],\n",
    "    eval_dataset=encoded_datasets[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=DataCollatorForMultipleChoice(tokenizer),\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "CdzABDVcIrJg"
   },
   "source": [
    "Finetuning via train method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma. If ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "/Users/zlxi/opt/anaconda3/envs/cs224n/lib/python3.7/site-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n",
      "***** Running training *****\n",
      "  Num examples = 396\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 75\n",
      "  Number of trainable parameters = 109483009\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4816d21a12bb477e97c5aab71ad085d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/75 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma. If ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 100\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86beebfba498490e970ecf3688cbd9bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6945326924324036, 'eval_accuracy': 0.3799999952316284, 'eval_runtime': 11.5765, 'eval_samples_per_second': 8.638, 'eval_steps_per_second': 0.605, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma. If ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 100\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "688551b812864bb89b5278a277633d66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6936256885528564, 'eval_accuracy': 0.36000001430511475, 'eval_runtime': 13.2679, 'eval_samples_per_second': 7.537, 'eval_steps_per_second': 0.528, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma. If ending0, pronoun, lemma, sentence, tag, definition, pos, ending1, id, fake_lemma are not expected by `BertForMultipleChoice.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 100\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "476a314bb2434f3285aa3c375936e2dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6935468912124634, 'eval_accuracy': 0.4099999964237213, 'eval_runtime': 13.6994, 'eval_samples_per_second': 7.3, 'eval_steps_per_second': 0.511, 'epoch': 3.0}\n",
      "{'train_runtime': 538.9478, 'train_samples_per_second': 2.204, 'train_steps_per_second': 0.139, 'train_loss': 0.6970101928710938, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=75, training_loss=0.6970101928710938, metrics={'train_runtime': 538.9478, 'train_samples_per_second': 2.204, 'train_steps_per_second': 0.139, 'train_loss': 0.6970101928710938, 'epoch': 3.0})"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Multiple choice on SWAG",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "cs224n",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}