MarathiRoberta.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of IndianLanguageModel.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "mount_file_id": "1Ug2op58U9UDsH6OVtozoh5R5vm5QTGoG",
      "authorship_tag": "ABX9TyOqzJEeTNPF/ZimdT6ojiQt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "cb60c4c0bce6405db40c9e5351362c96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f16521abb32841d7b94c366fa2d6fb6b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3376d0974ab34361b940046c7b2b2201",
              "IPY_MODEL_1f9ebebc24fe4b36ae99bfb3a1b9fa26"
            ]
          }
        },
        "f16521abb32841d7b94c366fa2d6fb6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3376d0974ab34361b940046c7b2b2201": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_00ae9ca17b44430a97778b7939e0d64f",
            "_dom_classes": [],
            "description": " 50%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 94011,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 47324,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc25664cb0c84906ba3e27dcecdebaeb"
          }
        },
        "1f9ebebc24fe4b36ae99bfb3a1b9fa26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_62a7d54cfa10446a855aef89769cea78",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 47324/94011 [10:57:20&lt;10:50:11,  1.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a003739027046789f4b5b76596e6c0c"
          }
        },
        "00ae9ca17b44430a97778b7939e0d64f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dc25664cb0c84906ba3e27dcecdebaeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "62a7d54cfa10446a855aef89769cea78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a003739027046789f4b5b76596e6c0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/deepampatel/randomnlp/blob/main/MarathiRoberta.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zEtQtwNqQyQr",
        "outputId": "c713c9d7-a1f8-4ea8-d59f-0bfb537d6fe1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fri Oct  9 06:57:39 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   41C    P0    29W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2qQQYX-F-WBN"
      },
      "source": [
        "## Download Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "32gyNAq-RMl3",
        "outputId": "79a1461e-a5e0-46c1-8cc2-8312013c2715",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "!wget https://oscar-public.huma-num.fr/shuffled/mr_dedup.txt.gz\n",
        "!gzip -d mr_dedup.txt.gz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-10-09 06:57:40--  https://oscar-public.huma-num.fr/shuffled/mr_dedup.txt.gz\n",
            "Resolving oscar-public.huma-num.fr (oscar-public.huma-num.fr)... 134.158.33.192\n",
            "Connecting to oscar-public.huma-num.fr (oscar-public.huma-num.fr)|134.158.33.192|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 313843639 (299M) [application/x-gzip]\n",
            "Saving to: ‘mr_dedup.txt.gz’\n",
            "\n",
            "mr_dedup.txt.gz     100%[===================>] 299.30M  27.6MB/s    in 12s     \n",
            "\n",
            "2020-10-09 06:57:52 (25.5 MB/s) - ‘mr_dedup.txt.gz’ saved [313843639/313843639]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UHainplJjrbr",
        "outputId": "6f8e2c7a-68b2-4333-a400-1efcd3140591",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VRUPX_5DUOzk"
      },
      "source": [
        "import os\n",
        "import gc\n",
        "import glob\n",
        "import torch\n",
        "import pickle\n",
        "import joblib\n",
        "from tqdm.auto import tqdm\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3GdaoH1b9Qxv",
        "outputId": "6cb4f23e-b5ff-4ccc-ddca-ee3df2fdb873",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install tokenizers\n",
        "!git clone https://github.com/huggingface/transformers\n",
        "!pip install transformers/.\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tokenizers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e9/ee/fedc3509145ad60fe5b418783f4a4c1b5462a4f0e8c7bbdbda52bdcda486/tokenizers-0.8.1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 1.7MB/s \n",
            "\u001b[?25hInstalling collected packages: tokenizers\n",
            "Successfully installed tokenizers-0.8.1\n",
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 22, done.\u001b[K\n",
            "remote: Counting objects: 100% (22/22), done.\u001b[K\n",
            "remote: Compressing objects: 100% (21/21), done.\u001b[K\n",
            "remote: Total 45802 (delta 4), reused 3 (delta 0), pack-reused 45780\u001b[K\n",
            "Receiving objects: 100% (45802/45802), 32.76 MiB | 28.77 MiB/s, done.\n",
            "Resolving deltas: 100% (31751/31751), done.\n",
            "Processing ./transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 1.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (1.18.5)\n",
            "Collecting tokenizers==0.9.0.rc2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/ad/ef09d13f98e42727c840863c4d7ba380eb041a6ca2f86413f6191c78162c/tokenizers-0.9.0rc2-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 9.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 19.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (20.4)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (3.12.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==3.3.1) (0.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.3.1) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.3.1) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.3.1) (0.16.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.3.1) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.3.1) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.3.1) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.3.1) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==3.3.1) (2.4.7)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers==3.3.1) (50.3.0)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-3.3.1-cp36-none-any.whl size=1133177 sha256=40975d70da28d40e091875da3513a1568930807888fb91fb1c2293413cd4b34a\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-6nxmc2d_/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=bbd4e7a34b91aa623a6eb391b31f7d9914ff268cfd54fb43f70a54d12c7fb13a\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sentencepiece, tokenizers, sacremoses, transformers\n",
            "  Found existing installation: tokenizers 0.8.1\n",
            "    Uninstalling tokenizers-0.8.1:\n",
            "      Successfully uninstalled tokenizers-0.8.1\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.9.0rc2 transformers-3.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YHtGKM2g-tsX"
      },
      "source": [
        "## Create the Tokenizer "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDR89kkYyIoV"
      },
      "source": [
        "from pathlib import Path\n",
        "\n",
        "from tokenizers import ByteLevelBPETokenizer\n",
        "\n",
        "paths = [\"mr_dedup.txt.gz\"]\n",
        "\n",
        "# Initialize a tokenizer\n",
        "tokenizer = ByteLevelBPETokenizer(\n",
        ")\n",
        "\n",
        "# Customize training\n",
        "tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=[\n",
        "    \"<s>\",\n",
        "    \"<pad>\",\n",
        "    \"</s>\",\n",
        "    \"<unk>\",\n",
        "    \"<mask>\",\n",
        "])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95V5NThny3gh",
        "outputId": "d427e5c6-438e-4352-b69a-5412dbf64dca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer.save_model(\"/content/drive/My Drive/marathiroberta\") #Saves it to drive"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['martbertochar/vocab.json', 'martbertochar/merges.txt']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sh29ZDhz-4br"
      },
      "source": [
        "## Load the saved Tokenizer and test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hEEgaNdizSz-"
      },
      "source": [
        "from tokenizers.implementations import ByteLevelBPETokenizer\n",
        "from tokenizers.processors import BertProcessing\n",
        "from tokenizers.decoders import ByteLevel\n",
        "\n",
        "tokenizer = ByteLevelBPETokenizer(\n",
        "    \"/content/drive/My Drive/marathiroberta/vocab.json\",\n",
        "    \"/content/drive/My Drive/marathiroberta/merges.txt\",\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FN9o8Q9UzfWf"
      },
      "source": [
        "tokenizer._tokenizer.post_processor = BertProcessing(\n",
        "    (\"</s>\", tokenizer.token_to_id(\"</s>\")),\n",
        "    (\"<s>\", tokenizer.token_to_id(\"<s>\")),\n",
        ")\n",
        "tokenizer.enable_truncation(max_length=512)\n",
        "op = tokenizer.encode(\"एखाद्या कुशल चित्रकाराने कुंचल्याच्या अवघ्या चार-सहा फटकाऱ्यांसरशी एखादे सुरेख चित्र निर्माण करावे तद्वत अवघ्या आठ ओळीत बालकवींनी एक सुंदर निसर्गचित्र शब्दांच्या कुंचल्याने या कवितेत रेखाटलेले आहे.  \")\n",
        "decoder = ByteLevel()\n",
        "tokenizer.decode(op)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UXKkiXcr_Hsz"
      },
      "source": [
        "## Create Roberta model and load the  tokenizer "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4kcDRRv05wMj"
      },
      "source": [
        "from transformers import RobertaConfig\n",
        "\n",
        "config = RobertaConfig(\n",
        "    vocab_size=52_000,\n",
        "    max_position_embeddings=514,\n",
        "    num_attention_heads=12,\n",
        "    num_hidden_layers=12,\n",
        "    type_vocab_size=1,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PkEqh815-9g"
      },
      "source": [
        "from transformers import RobertaTokenizerFast\n",
        "\n",
        "\n",
        "tokenizer = RobertaTokenizerFast.from_pretrained(\"/content/drive/My Drive/marathiroberta\", max_len=512)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u2ZY6XZHaNPS",
        "outputId": "829939da-c276-4f4f-9bb6-c2fe31bbb328",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "tokenizer.save_pretrained(\"/content/drive/My Drive/marathiroberta\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/drive/My Drive/tokenizer/tokenizer_config.json',\n",
              " '/content/drive/My Drive/tokenizer/special_tokens_map.json',\n",
              " ('/content/drive/My Drive/tokenizer/vocab.json',\n",
              "  '/content/drive/My Drive/tokenizer/merges.txt'),\n",
              " '/content/drive/My Drive/tokenizer/added_tokens.json')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-XlomtCE6BMM"
      },
      "source": [
        "from transformers import RobertaForMaskedLM\n",
        "\n",
        "model = RobertaForMaskedLM(config=config)\n",
        "# model = RobertaForMaskedLM.from_pretrained('/content/drive/My Drive/marathiroberta/checkpoint-90000', return_dict=True)  #To load from a checkpoint"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Gn9AEbg6CZX",
        "outputId": "b2eb70c2-e518-459d-b299-06c669b5f005",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "model.num_parameters()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "126031648"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MauBukjx_taQ"
      },
      "source": [
        "## Create a lazy data loader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HiOhXkZCk4Vf"
      },
      "source": [
        "from torch.utils.data.dataset import Dataset\n",
        "import os\n",
        "from transformers.tokenization_utils import PreTrainedTokenizer\n",
        "import torch\n",
        "import logging\n",
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "class LazyLineByLineTextDataset(Dataset):\n",
        "    \"\"\"\n",
        "    This will be superseded by a framework-agnostic approach\n",
        "    soon.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: int):\n",
        "        assert os.path.isfile(file_path), f\"Input file path {file_path} not found\"\n",
        "        # Here, we do not cache the features, operating under the assumption\n",
        "        # that we will soon use fast multithreaded tokenizers from the\n",
        "        # `tokenizers` repo everywhere =)\n",
        "        logger.info(\"Creating features from dataset file at %s\", file_path)\n",
        "\n",
        "        with open(file_path, encoding=\"utf-8\") as f:\n",
        "            lines = [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]\n",
        "        self.block_size = block_size\n",
        "        # batch_encoding = tokenizer(lines, add_special_tokens=True, truncation=True, max_length=block_size)\n",
        "        self.examples = lines\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.examples)\n",
        "\n",
        "    def __getitem__(self, i) -> torch.Tensor:\n",
        "        \n",
        "        return torch.tensor(self.tokenizer(self.examples[i], add_special_tokens=True, truncation=True, max_length=self.block_size)[\"input_ids\"], dtype=torch.long)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fUFHCEBV_63Z"
      },
      "source": [
        "## Create a dataloader and use that to create a trainer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3yJ1NdpR6Ddz",
        "outputId": "004a1d06-5632-48d2-aa63-1a293d7a5775",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "%%time\n",
        "from transformers import LineByLineTextDataset\n",
        "\n",
        "#For lazy loading the data\n",
        "dataset = LazyLineByLineTextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "    file_path=\"mr_dedup.txt\",\n",
        "    block_size=128,\n",
        ")\n",
        "\n",
        "#For loading the whole dataset in memory\n",
        "# dataset = LineByLineTextDataset(\n",
        "#     tokenizer=tokenizer,\n",
        "#     file_path=\"mr_dedup.txt\",\n",
        "#     block_size=128,\n",
        "# )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 8.97 s, sys: 4.46 s, total: 13.4 s\n",
            "Wall time: 13.4 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dThQR1ms6HWp"
      },
      "source": [
        "from transformers import DataCollatorForLanguageModeling\n",
        "\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer, mlm=True, mlm_probability=0.20\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqH0KGE76Iue",
        "outputId": "3b278286-b3a8-4979-a685-c9bd2a7c3486",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"/content/drive/My Drive/marathiroberta\",\n",
        "    overwrite_output_dir=True,\n",
        "    num_train_epochs=3,\n",
        "    per_gpu_train_batch_size=48,\n",
        "    save_steps=5000,\n",
        "    save_total_limit=1,\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=dataset,\n",
        "    prediction_loss_only=True,\n",
        ")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/trainer.py:246: FutureWarning: Passing `prediction_loss_only` as a keyword argument is deprecated and won't be possible in a future version. Use `args.prediction_loss_only` instead. Setting `args.prediction_loss_only=True\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ti-0tsRk6S_e",
        "outputId": "ba1a9f5b-3513-4739-ae65-ab99b6129f73",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "cb60c4c0bce6405db40c9e5351362c96",
            "f16521abb32841d7b94c366fa2d6fb6b",
            "3376d0974ab34361b940046c7b2b2201",
            "1f9ebebc24fe4b36ae99bfb3a1b9fa26",
            "00ae9ca17b44430a97778b7939e0d64f",
            "dc25664cb0c84906ba3e27dcecdebaeb",
            "62a7d54cfa10446a855aef89769cea78",
            "6a003739027046789f4b5b76596e6c0c"
          ]
        }
      },
      "source": [
        "%%time\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cb60c4c0bce6405db40c9e5351362c96",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=94011.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "{'loss': 0.4169935302734375, 'learning_rate': 4.9734073672229845e-05, 'epoch': 0.015955579666209273}\n",
            "{'loss': 0.4232584228515625, 'learning_rate': 4.9468147344459694e-05, 'epoch': 0.031911159332418546}\n",
            "{'loss': 0.4242197265625, 'learning_rate': 4.9202221016689536e-05, 'epoch': 0.04786673899862782}\n",
            "{'loss': 0.4336846923828125, 'learning_rate': 4.8936294688919385e-05, 'epoch': 0.06382231866483709}\n",
            "{'loss': 0.4378968505859375, 'learning_rate': 4.8670368361149234e-05, 'epoch': 0.07977789833104637}\n",
            "{'loss': 0.4377626953125, 'learning_rate': 4.840444203337908e-05, 'epoch': 0.09573347799725564}\n",
            "{'loss': 0.443458740234375, 'learning_rate': 4.813851570560892e-05, 'epoch': 0.11168905766346492}\n",
            "{'loss': 0.442999267578125, 'learning_rate': 4.787258937783877e-05, 'epoch': 0.12764463732967418}\n",
            "{'loss': 0.475087890625, 'learning_rate': 4.760666305006861e-05, 'epoch': 0.14360021699588346}\n",
            "{'loss': 0.503595703125, 'learning_rate': 4.734073672229846e-05, 'epoch': 0.15955579666209274}\n",
            "{'loss': 0.4983759765625, 'learning_rate': 4.70748103945283e-05, 'epoch': 0.175511376328302}\n",
            "{'loss': 0.49851171875, 'learning_rate': 4.6808884066758144e-05, 'epoch': 0.19146695599451127}\n",
            "{'loss': 0.50030810546875, 'learning_rate': 4.654295773898799e-05, 'epoch': 0.20742253566072055}\n",
            "{'loss': 0.4959833984375, 'learning_rate': 4.6277031411217835e-05, 'epoch': 0.22337811532692983}\n",
            "{'loss': 0.49774658203125, 'learning_rate': 4.6011105083447685e-05, 'epoch': 0.2393336949931391}\n",
            "{'loss': 0.4992041015625, 'learning_rate': 4.5745178755677534e-05, 'epoch': 0.25528927465934836}\n",
            "{'loss': 0.5026337890625, 'learning_rate': 4.5479252427907376e-05, 'epoch': 0.27124485432555767}\n",
            "{'loss': 0.50426123046875, 'learning_rate': 4.521332610013722e-05, 'epoch': 0.2872004339917669}\n",
            "{'loss': 0.504654296875, 'learning_rate': 4.494739977236707e-05, 'epoch': 0.3031560136579762}\n",
            "{'loss': 0.5088671875, 'learning_rate': 4.468147344459691e-05, 'epoch': 0.3191115933241855}\n",
            "{'loss': 0.5110693359375, 'learning_rate': 4.441554711682676e-05, 'epoch': 0.33506717299039473}\n",
            "{'loss': 0.5122822265625, 'learning_rate': 4.41496207890566e-05, 'epoch': 0.351022752656604}\n",
            "{'loss': 0.5086982421875, 'learning_rate': 4.3883694461286443e-05, 'epoch': 0.3669783323228133}\n",
            "{'loss': 0.5145029296875, 'learning_rate': 4.361776813351629e-05, 'epoch': 0.38293391198902255}\n",
            "{'loss': 0.508921875, 'learning_rate': 4.335184180574614e-05, 'epoch': 0.39888949165523185}\n",
            "{'loss': 0.51350390625, 'learning_rate': 4.3085915477975984e-05, 'epoch': 0.4148450713214411}\n",
            "{'loss': 0.5150322265625, 'learning_rate': 4.281998915020583e-05, 'epoch': 0.43080065098765036}\n",
            "{'loss': 0.513615234375, 'learning_rate': 4.2554062822435675e-05, 'epoch': 0.44675623065385967}\n",
            "{'loss': 0.5221572265625, 'learning_rate': 4.228813649466552e-05, 'epoch': 0.4627118103200689}\n",
            "{'loss': 0.51719140625, 'learning_rate': 4.202221016689537e-05, 'epoch': 0.4786673899862782}\n",
            "{'loss': 0.519068359375, 'learning_rate': 4.175628383912521e-05, 'epoch': 0.4946229696524875}\n",
            "{'loss': 0.523765625, 'learning_rate': 4.149035751135505e-05, 'epoch': 0.5105785493186967}\n",
            "{'loss': 0.5220947265625, 'learning_rate': 4.12244311835849e-05, 'epoch': 0.526534128984906}\n",
            "{'loss': 0.5289111328125, 'learning_rate': 4.095850485581474e-05, 'epoch': 0.5424897086511153}\n",
            "{'loss': 0.521685546875, 'learning_rate': 4.069257852804459e-05, 'epoch': 0.5584452883173245}\n",
            "{'loss': 0.52837109375, 'learning_rate': 4.042665220027444e-05, 'epoch': 0.5744008679835338}\n",
            "{'loss': 0.520361328125, 'learning_rate': 4.016072587250428e-05, 'epoch': 0.5903564476497432}\n",
            "{'loss': 0.521404296875, 'learning_rate': 3.989479954473413e-05, 'epoch': 0.6063120273159524}\n",
            "{'loss': 0.530630859375, 'learning_rate': 3.9628873216963975e-05, 'epoch': 0.6222676069821617}\n",
            "{'loss': 0.5273671875, 'learning_rate': 3.936294688919382e-05, 'epoch': 0.638223186648371}\n",
            "{'loss': 0.55281640625, 'learning_rate': 3.9097020561423666e-05, 'epoch': 0.6541787663145802}\n",
            "{'loss': 0.5492578125, 'learning_rate': 3.883109423365351e-05, 'epoch': 0.6701343459807895}\n",
            "{'loss': 0.546775390625, 'learning_rate': 3.856516790588335e-05, 'epoch': 0.6860899256469988}\n",
            "{'loss': 0.5470625, 'learning_rate': 3.829924157811321e-05, 'epoch': 0.702045505313208}\n",
            "{'loss': 0.549072265625, 'learning_rate': 3.803331525034305e-05, 'epoch': 0.7180010849794173}\n",
            "{'loss': 0.544533203125, 'learning_rate': 3.776738892257289e-05, 'epoch': 0.7339566646456266}\n",
            "{'loss': 0.549470703125, 'learning_rate': 3.750146259480274e-05, 'epoch': 0.7499122443118359}\n",
            "{'loss': 0.548189453125, 'learning_rate': 3.723553626703258e-05, 'epoch': 0.7658678239780451}\n",
            "{'loss': 0.550830078125, 'learning_rate': 3.696960993926243e-05, 'epoch': 0.7818234036442544}\n",
            "{'loss': 0.555353515625, 'learning_rate': 3.6703683611492274e-05, 'epoch': 0.7977789833104637}\n",
            "{'loss': 0.557841796875, 'learning_rate': 3.6437757283722116e-05, 'epoch': 0.8137345629766729}\n",
            "{'loss': 0.552779296875, 'learning_rate': 3.6171830955951966e-05, 'epoch': 0.8296901426428822}\n",
            "{'loss': 0.551322265625, 'learning_rate': 3.590590462818181e-05, 'epoch': 0.8456457223090915}\n",
            "{'loss': 0.55927734375, 'learning_rate': 3.563997830041166e-05, 'epoch': 0.8616013019753007}\n",
            "{'loss': 0.56312890625, 'learning_rate': 3.5374051972641506e-05, 'epoch': 0.87755688164151}\n",
            "{'loss': 0.55178515625, 'learning_rate': 3.510812564487135e-05, 'epoch': 0.8935124613077193}\n",
            "{'loss': 0.55836328125, 'learning_rate': 3.484219931710119e-05, 'epoch': 0.9094680409739286}\n",
            "{'loss': 0.55678125, 'learning_rate': 3.457627298933104e-05, 'epoch': 0.9254236206401378}\n",
            "{'loss': 0.5593984375, 'learning_rate': 3.431034666156088e-05, 'epoch': 0.9413792003063471}\n",
            "{'loss': 0.56642578125, 'learning_rate': 3.4044420333790724e-05, 'epoch': 0.9573347799725564}\n",
            "{'loss': 0.569708984375, 'learning_rate': 3.3778494006020573e-05, 'epoch': 0.9732903596387656}\n",
            "{'loss': 0.5634921875, 'learning_rate': 3.3512567678250416e-05, 'epoch': 0.989245939304975}\n",
            "{'loss': 0.57623046875, 'learning_rate': 3.3246641350480265e-05, 'epoch': 1.0052015189711843}\n",
            "{'loss': 0.60802734375, 'learning_rate': 3.2980715022710114e-05, 'epoch': 1.0211570986373935}\n",
            "{'loss': 0.6096484375, 'learning_rate': 3.2714788694939956e-05, 'epoch': 1.0371126783036029}\n",
            "{'loss': 0.60880078125, 'learning_rate': 3.2448862367169805e-05, 'epoch': 1.053068257969812}\n",
            "{'loss': 0.6070703125, 'learning_rate': 3.218293603939965e-05, 'epoch': 1.0690238376360213}\n",
            "{'loss': 0.61722265625, 'learning_rate': 3.191700971162949e-05, 'epoch': 1.0849794173022307}\n",
            "{'loss': 0.61996484375, 'learning_rate': 3.165108338385934e-05, 'epoch': 1.1009349969684399}\n",
            "Buffered data was truncated after reaching the output size limit."
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "49IhJyxg6W9R"
      },
      "source": [
        "trainer.save_model(\"/content/drive/My Drive/marathiroberta\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rT-ZKunMACOB"
      },
      "source": [
        "## Load saved model and test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wh_uqkq3IVts"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "fill_mask = pipeline(\n",
        "    \"fill-mask\",\n",
        "    model=\"/content/drive/My Drive/marathiroberta\",\n",
        "    tokenizer=\"/content/drive/My Drive/marathiroberta\"\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c2mlD66uNp_z",
        "outputId": "c15d13d8-b217-43c0-df6e-5e4f9cfa97e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "fill_mask(\"राष्ट्रवादीचे सर्वेसर्वा <mask> पवार यांनी मुंडेंना राष्ट्रवादीचा प्रस्ताव नसल्याचे म्हटले आहे.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'score': 0.9996238946914673,\n",
              "  'sequence': '<s>राष्ट्रवादीचे सर्वेसर्वा शरद पवार यांनी मुंडेंना राष्ट्रवादीचा प्रस्ताव नसल्याचे म्हटले आहे.</s>',\n",
              "  'token': 1405,\n",
              "  'token_str': 'Ġà¤¶à¤°à¤¦'},\n",
              " {'score': 0.00015381297271233052,\n",
              "  'sequence': '<s>राष्ट्रवादीचे सर्वेसर्वाशरद पवार यांनी मुंडेंना राष्ट्रवादीचा प्रस्ताव नसल्याचे म्हटले आहे.</s>',\n",
              "  'token': 4942,\n",
              "  'token_str': 'à¤¶à¤°à¤¦'},\n",
              " {'score': 8.309441909659654e-05,\n",
              "  'sequence': '<s>राष्ट्रवादीचे सर्वेसर्वा भरत पवार यांनी मुंडेंना राष्ट्रवादीचा प्रस्ताव नसल्याचे म्हटले आहे.</s>',\n",
              "  'token': 1207,\n",
              "  'token_str': 'Ġà¤Ńà¤°à¤¤'},\n",
              " {'score': 4.730443106382154e-05,\n",
              "  'sequence': '<s>राष्ट्रवादीचे सर्वेसर्वा छगन पवार यांनी मुंडेंना राष्ट्रवादीचा प्रस्ताव नसल्याचे म्हटले आहे.</s>',\n",
              "  'token': 3803,\n",
              "  'token_str': 'Ġà¤Ľà¤Ĺà¤¨'},\n",
              " {'score': 2.4680019123479724e-05,\n",
              "  'sequence': '<s>राष्ट्रवादीचे सर्वेसर्वा करण पवार यांनी मुंडेंना राष्ट्रवादीचा प्रस्ताव नसल्याचे म्हटले आहे.</s>',\n",
              "  'token': 345,\n",
              "  'token_str': 'Ġà¤ķà¤°à¤£'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    }
  ]
}