Text_Classification_with_Hugging_Face_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text Classification with Hugging Face Transformers.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7eavGw5rFoIw"
      },
      "outputs": [],
      "source": [
        "!pip install ktrain"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%matplotlib inline\n",
        "import os\n",
        "os.environ['CUDA_DEVICE_ORDER']=\"PCI_BUS_ID\";\n",
        "os.environ['CUDA_VISIBLE_DEVICES']=\"0\";"
      ],
      "metadata": {
        "id": "MiUSEZUAFz2U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import ktrain\n",
        "from ktrain import text\n",
        "from sklearn.datasets import fetch_20newsgroups"
      ],
      "metadata": {
        "id": "6cQQkYsJGSEj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "categories = ['alt.atheism', \n",
        "              'soc.religion.christian', \n",
        "              'comp.graphics',\n",
        "              'sci.med',\n",
        "              'rec.sport.baseball']"
      ],
      "metadata": {
        "id": "SGbepo2VGZ6U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train = fetch_20newsgroups(\n",
        "    subset = 'train',\n",
        "    categories = categories,\n",
        "    shuffle = True,\n",
        "    random_state =0\n",
        ")"
      ],
      "metadata": {
        "id": "vMiBmLR6KfPX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test = fetch_20newsgroups(\n",
        "    subset = 'test',\n",
        "    categories = categories,\n",
        "    shuffle = True,\n",
        "    random_state=0\n",
        ")"
      ],
      "metadata": {
        "id": "YuogzKfmKr-T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zz5Clo9OOYuu",
        "outputId": "eb930803-4c5e-4633-cbe7-142a219a46d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "IOPub data rate exceeded.\n",
            "The notebook server will temporarily stop sending output\n",
            "to the client in order to avoid crashing it.\n",
            "To change this limit, set the config variable\n",
            "`--NotebookApp.iopub_data_rate_limit`.\n",
            "\n",
            "Current values:\n",
            "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
            "NotebookApp.rate_limit_window=3.0 (secs)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test.keys()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zm2gBz-GPbrj",
        "outputId": "48917802-6af8-4910-93ec-47b3c9d2b8e7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['data', 'filenames', 'target_names', 'target', 'DESCR'])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test.target"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_ocO7EL5QY3a",
        "outputId": "beed04e3-252d-4f53-a271-949e8ff7a15f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 4, 2, ..., 2, 3, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test.target_names"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1iCZ-x_IQw6T",
        "outputId": "9b2e080d-e329-42bf-c392-aa7e487c7ea6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['alt.atheism',\n",
              " 'comp.graphics',\n",
              " 'rec.sport.baseball',\n",
              " 'sci.med',\n",
              " 'soc.religion.christian']"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_train = train.data\n",
        "y_train = train.target\n",
        "\n",
        "X_test = test.data\n",
        "y_test = test.target"
      ],
      "metadata": {
        "id": "JL2NE78ZQypw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(X_train), len(X_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C2h5-wggQ67i",
        "outputId": "84c60d19-6f68-4ca1-aad2-15cf19fa4100"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2854, 1899)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#X_test"
      ],
      "metadata": {
        "id": "Y9t9FRi1RFKr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build ML model with Transformer"
      ],
      "metadata": {
        "id": "S7Cpf1OGRTxa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = \"distilbert-base-uncased\"\n",
        "\n",
        "trans = text.Transformer(model_name, maxlen=512, class_names=categories)"
      ],
      "metadata": {
        "id": "aebL1RW9RJ9N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = trans.preprocess_train(X_train,y_train)\n",
        "test_data = trans.preprocess_test(X_test, y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "id": "iskKehXNR5op",
        "outputId": "c66e04fb-5925-4078-dce4-d77ac47ec75f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 291\n",
            "\t95percentile : 820\n",
            "\t99percentile : 1757\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              ""
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 323\n",
            "\t95percentile : 894\n",
            "\t99percentile : 2394\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "<style>\n",
              "    /* Turns off some styling */\n",
              "    progress {\n",
              "        /* gets rid of default border in Firefox and Opera. */\n",
              "        border: none;\n",
              "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "        background-size: auto;\n",
              "    }\n",
              "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "        background: #F44336;\n",
              "    }\n",
              "</style>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              ""
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = trans.get_classifier()"
      ],
      "metadata": {
        "id": "KQ9ofHRpSPzH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "learner = ktrain.get_learner(model, train_data=train_data, \n",
        "                             val_data = test_data,\n",
        "                             batch_size = 16)"
      ],
      "metadata": {
        "id": "PI3Of_MPSV3l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# to find the best learning rate"
      ],
      "metadata": {
        "id": "0A43IbmAWNOQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learner.lr_find(show_plot=True, max_epochs=10)"
      ],
      "metadata": {
        "id": "GMmYu9HFSg77"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# only fit one epoch\n",
        "\n",
        "learner.fit_onecycle(1e-4,1)\n",
        "#.fit_onecycle(1e-4, 1)\n",
        "# 0.004\n",
        "# 1 epoch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L0pHWEIcTGnM",
        "outputId": "3a39bcf3-4552-4a52-a61f-431bb74b7c69"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 0.0001...\n",
            "179/179 [==============================] - 408s 2s/step - loss: 0.4956 - accuracy: 0.8577 - val_loss: 0.2057 - val_accuracy: 0.9400\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f4187f48910>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# confusion matrix\n",
        "learner.validate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JeQVAfCOTbkM",
        "outputId": "08a3d38f-5ea0-4a81-a2ca-95f170db8d5f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.85      0.93      0.89       319\n",
            "           1       0.91      0.98      0.95       389\n",
            "           2       1.00      0.96      0.98       397\n",
            "           3       0.97      0.93      0.95       396\n",
            "           4       0.96      0.90      0.93       398\n",
            "\n",
            "    accuracy                           0.94      1899\n",
            "   macro avg       0.94      0.94      0.94      1899\n",
            "weighted avg       0.94      0.94      0.94      1899\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[296,   8,   0,   4,  11],\n",
              "       [  1, 382,   0,   5,   1],\n",
              "       [  6,   9, 380,   1,   1],\n",
              "       [  9,  18,   1, 367,   1],\n",
              "       [ 36,   2,   0,   0, 360]])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "learner.view_top_losses(n=5, preproc=trans)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KGsmGkN2TsUb",
        "outputId": "b823d207-b065-49ba-901b-fac99515eee0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------\n",
            "id:787 | loss:6.06 | true:comp.graphics | pred:soc.religion.christian)\n",
            "\n",
            "----------\n",
            "id:908 | loss:5.61 | true:comp.graphics | pred:soc.religion.christian)\n",
            "\n",
            "----------\n",
            "id:562 | loss:5.54 | true:rec.sport.baseball | pred:soc.religion.christian)\n",
            "\n",
            "----------\n",
            "id:238 | loss:5.54 | true:comp.graphics | pred:soc.religion.christian)\n",
            "\n",
            "----------\n",
            "id:170 | loss:5.52 | true:comp.graphics | pred:alt.atheism)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "8w0jfq67Uh9h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# predict on new data"
      ],
      "metadata": {
        "id": "nUW7kavWUsyi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "predictor = ktrain.get_predictor(learner.model, preproc = trans)"
      ],
      "metadata": {
        "id": "fAeWV7_ZUooT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = 'Jesus Christ is the central figure of Christianity'"
      ],
      "metadata": {
        "id": "OsSXAMLOUoty"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predictor.predict(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Q-AekBUNVLtL",
        "outputId": "036501e9-75dc-47fa-dd6e-1cadea75aa35"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'rec.sport.baseball'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictor.explain(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vn_ZgVDhUowL",
        "outputId": "ea70a854-233f-4426-ee49-db3ced0deaf0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ktrain/text/predictor.py:124: UserWarning: ktrain requires a forked version of eli5 to support tf.keras. Install with: pip install https://github.com/amaiya/eli5/archive/refs/heads/tfkeras_0_10_1.zip\n",
            "  warnings.warn(msg)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictor.save('model')"
      ],
      "metadata": {
        "id": "czI9UgArUoy1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "2LSLWf06Uo1i"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}