bangla_tokenizer(1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91d38bcd-23c8-4101-9c86-f418943f1c46",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "392aae20-4e83-4ae0-9f96-a2b15daf8679",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a bangla tokenizer \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf8f3bf6-123b-44aa-8f22-de75c09b536f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#training the llama model on the bangla dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "47e3a570-f86f-49d4-b29a-57bb7c600ab9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b420eea3aed54113ae69532760503e39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "51ce96bd-bf0e-4ebc-a663-5ec1a6c868bd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting unsloth@ git+https://github.com/unslothai/unsloth.git (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Cloning https://github.com/unslothai/unsloth.git to /tmp/pip-install-sgd413wv/unsloth_65310b1d2af94f05add33144e223b344\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/unslothai/unsloth.git /tmp/pip-install-sgd413wv/unsloth_65310b1d2af94f05add33144e223b344\n",
      "  Resolved https://github.com/unslothai/unsloth.git to commit bfe38e6ea8d3d7cf8ce9e37962de03c71c90cbe2\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: packaging in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (23.2)\n",
      "Requirement already satisfied: tyro in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.7.3)\n",
      "Requirement already satisfied: transformers>=4.43.2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (4.44.0.dev0)\n",
      "Requirement already satisfied: datasets>=2.16.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.19.1)\n",
      "Requirement already satisfied: sentencepiece>=0.2.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.2.0)\n",
      "Requirement already satisfied: tqdm in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (4.66.2)\n",
      "Requirement already satisfied: psutil in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (5.9.8)\n",
      "Collecting wheel>=0.42.0 (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading wheel-0.44.0-py3-none-any.whl.metadata (2.3 kB)\n",
      "Requirement already satisfied: numpy in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.26.4)\n",
      "Collecting protobuf<4.0.0 (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (679 bytes)\n",
      "Requirement already satisfied: huggingface-hub in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.24.5)\n",
      "Requirement already satisfied: hf-transfer in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.1.6)\n",
      "Requirement already satisfied: filelock in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.13.3)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (15.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.3.8)\n",
      "Requirement already satisfied: pandas in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.2.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.31.0)\n",
      "Requirement already satisfied: xxhash in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2024.2.0)\n",
      "Requirement already satisfied: aiohttp in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.9.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (6.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from huggingface-hub->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (4.10.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers>=4.43.2->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2023.12.25)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers>=4.43.2->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.19.1)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers>=4.43.2->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.4.2)\n",
      "Requirement already satisfied: docstring-parser>=0.14.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.16)\n",
      "Requirement already satisfied: rich>=11.1.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (13.7.1)\n",
      "Requirement already satisfied: shtab>=1.5.6 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.7.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (4.0.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2024.2.2)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from rich>=11.1.0->tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from rich>=11.1.0->tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.17.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2024.1)\n",
      "Requirement already satisfied: mdurl~=0.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.16.0->unsloth@ git+https://github.com/unslothai/unsloth.git->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.16.0)\n",
      "Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading wheel-0.44.0-py3-none-any.whl (67 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.1/67.1 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hBuilding wheels for collected packages: unsloth\n",
      "  Building wheel for unsloth (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for unsloth: filename=unsloth-2024.8-py3-none-any.whl size=136830 sha256=a6bbeda8461ba5b6da01f2774dc8101037dad4c1d25a934f866a7fa1fb30f391\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-rkfvqe_x/wheels/ed/d4/e9/76fb290ee3df0a5fc21ce5c2c788e29e9607a2353d8342fd0d\n",
      "Successfully built unsloth\n",
      "Installing collected packages: wheel, unsloth, protobuf\n",
      "  Attempting uninstall: wheel\n",
      "    Found existing installation: wheel 0.41.2\n",
      "    Uninstalling wheel-0.41.2:\n",
      "      Successfully uninstalled wheel-0.41.2\n",
      "  Attempting uninstall: protobuf\n",
      "    Found existing installation: protobuf 4.25.3\n",
      "    Uninstalling protobuf-4.25.3:\n",
      "      Successfully uninstalled protobuf-4.25.3\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "axolotl 0.4.1 requires peft==0.11.1, but you have peft 0.12.1.dev0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed protobuf-3.20.3 unsloth-2024.8 wheel-0.44.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: xformers<0.0.27 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (0.0.23.post1+cu118)\n",
      "Collecting trl<0.9.0\n",
      "  Downloading trl-0.8.6-py3-none-any.whl.metadata (11 kB)\n",
      "Requirement already satisfied: peft in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (0.12.1.dev0)\n",
      "Requirement already satisfied: accelerate in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (0.32.0)\n",
      "Requirement already satisfied: bitsandbytes in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (0.43.3)\n",
      "Requirement already satisfied: datasets in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (2.19.1)\n",
      "Requirement already satisfied: transformers in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (4.44.0.dev0)\n",
      "Downloading trl-0.8.6-py3-none-any.whl (245 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m245.2/245.2 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: trl\n",
      "  Attempting uninstall: trl\n",
      "    Found existing installation: trl 0.9.6\n",
      "    Uninstalling trl-0.9.6:\n",
      "      Successfully uninstalled trl-0.9.6\n",
      "Successfully installed trl-0.8.6\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# # #download unsloth\n",
    "# !pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
    "# !pip install --no-deps \"xformers<0.0.27\" \"trl<0.9.0\" peft accelerate bitsandbytes datasets transformers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0bbe24c0-8785-40c9-9bd6-1edf1428cefc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (2.19.1)\n",
      "Requirement already satisfied: tokenizers in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (0.19.1)\n",
      "Requirement already satisfied: transformers[sentencepiece] in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (4.44.0.dev0)\n",
      "Requirement already satisfied: filelock in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (3.13.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (1.26.4)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (15.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: pandas in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (2.2.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (4.66.2)\n",
      "Requirement already satisfied: xxhash in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets) (2024.2.0)\n",
      "Requirement already satisfied: aiohttp in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (3.9.3)\n",
      "Requirement already satisfied: huggingface-hub>=0.21.2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (0.24.5)\n",
      "Requirement already satisfied: packaging in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers[sentencepiece]) (2023.12.25)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.4.2)\n",
      "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.2.0)\n",
      "Requirement already satisfied: protobuf in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from transformers[sentencepiece]) (3.20.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from huggingface-hub>=0.21.2->datasets) (4.10.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from pandas->datasets) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from pandas->datasets) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from pandas->datasets) (2024.1)\n",
      "Requirement already satisfied: six>=1.5 in /root/miniconda3/envs/py3.10/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# !pip install datasets tokenizers transformers[sentencepiece]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "68c14543-11a7-4a79-9a23-45c975177aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from tokenizers.normalizers import NFD, StripAccents, Replace, Sequence\n",
    "from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, trainers, processors, decoders\n",
    "import re\n",
    "from tokenizers import Regex\n",
    "from tokenizers.normalizers import NFD, StripAccents, Replace, Sequence\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9501f3d1-bc97-4bda-8856-e12bc155abe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "from transformers import PreTrainedTokenizerFast\n",
    "from datasets import load_dataset\n",
    "from tokenizers import Tokenizer\n",
    "from transformers import DataCollatorForLanguageModeling, Trainer, TrainingArguments\n",
    "from huggingface_hub import HfApi, list_models\n",
    "import torch\n",
    "from unsloth import FastLanguageModel\n",
    "from tqdm import tqdm\n",
    "from huggingface_hub import login\n",
    "from peft import get_peft_model, LoraConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "494f2b7f-23bf-4e2a-9bef-f3272ba63195",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting flash-attn>=2.6.3\n",
      "  Downloading flash_attn-2.6.3.tar.gz (2.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m38.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hBuilding wheels for collected packages: flash-attn\n",
      "  Building wheel for flash-attn (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for flash-attn: filename=flash_attn-2.6.3-cp310-cp310-linux_x86_64.whl size=188977699 sha256=d16f6008f2b7727718653bfd087a20f3e18e2a4c8e91b9f38da2439f3b0bf529\n",
      "  Stored in directory: /root/.cache/pip/wheels/7e/e3/c3/89c7a2f3c4adc07cd1c675f8bb7b9ad4d18f64a72bccdfe826\n",
      "Successfully built flash-attn\n",
      "Installing collected packages: flash-attn\n",
      "  Attempting uninstall: flash-attn\n",
      "    Found existing installation: flash-attn 2.5.5\n",
      "    Uninstalling flash-attn-2.5.5:\n",
      "      Successfully uninstalled flash-attn-2.5.5\n",
      "Successfully installed flash-attn-2.6.3\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install --no-deps --upgrade \"flash-attn>=2.6.3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "51051ef8-5cd6-4f84-8454-80a7669684ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/datasets/load.py:1486: FutureWarning: The repository for oscar contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/oscar\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36d74cd6a6ae44a0bdf0a5b7d9c73ec1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/14.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36ce094fc8004a54959f59720da22ffe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/303k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef57df1f35234c3d838ba7a2d0ec5554",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/328 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fe2a9c8f7c34808a8515d2c6e0a4b16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/390M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00513e0a90f94e2cb88649510a2a8a4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/390M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d5f27a2ef0a42c9bb7d04978fe9731f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/391M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5a25227590b44bd8147cfa6b265c9c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/86.2M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a58f1f5a4db496192b06fef9d0f2b61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/1114481 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# load bangla dataset\n",
    "dataset = load_dataset(\"oscar\", \"unshuffled_deduplicated_bn\", split=\"train\")\n",
    "\n",
    "#link to dataset  = https://huggingface.co/datasets/oscar-corpus/oscar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a08ae0c9-67f2-44d1-a331-5b2005cb3d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_training_corpus():\n",
    "    for i in range(0, len(dataset), 1000):\n",
    "        yield dataset[i : i + 1000][\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9196f38a-50d1-4fbc-8223-36ce414c2bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# BPE tokenizer \n",
    "tokenizer = Tokenizer(models.BPE())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c4981263-25c1-40c5-addf-e80d6c73a2ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.normalizer = normalizers.Sequence([\n",
    "    normalizers.NFD(),\n",
    "    normalizers.StripAccents(),\n",
    "    normalizers.Replace(Regex(r\"[\\s]+\"), \" \"),\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fa677e99-daa2-44a7-815c-b6d9cd6c99d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set normalizer\n",
    "tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "037660ed-eaf3-45a2-8761-91d01d870978",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the trainer\n",
    "trainer = trainers.BpeTrainer(vocab_size=30000, special_tokens=[\"<s>\", \"<pad>\", \"</s>\", \"<unk>\", \"<mask>\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0040bf1-d538-4f07-91de-1ac60a2f4008",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the tokenizer\n",
    "tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)\n",
    "\n",
    "# Save\n",
    "tokenizer.save(\"bangla_tokenizer.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "02be0a59-4b12-428f-b1e1-75f80ac6a089",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load\n",
    "bangla_tokenizer = Tokenizer.from_file(\"bangla_tokenizer.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0c6d55e5-cc07-4a98-8a8d-9a74c24f9bb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wrap the tokenizer\n",
    "bangla_tokenizer = PreTrainedTokenizerFast(tokenizer_object=bangla_tokenizer, unk_token=\"<unk>\", pad_token=\"<pad>\", cls_token=\"<s>\", sep_token=\"</s>\", mask_token=\"<mask>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "feae6ee9-5d7a-475c-9f1f-c800a172ae18",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_dataset(examples):\n",
    "    texts = examples[\"text\"]\n",
    "    # Tokenize the texts\n",
    "    tokenized_texts = bangla_tokenizer(texts, padding=\"max_length\", truncation=True, max_length=512)\n",
    "    return {\"input_ids\": tokenized_texts[\"input_ids\"], \"attention_mask\": tokenized_texts[\"attention_mask\"]}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3bb28b7d-9b35-4fe6-952d-6f9d6988399d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/datasets/load.py:1486: FutureWarning: The repository for oscar contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/oscar\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83bd63e09e7b499587d9b59c853b7e46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/11145 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load and format the dataset\n",
    "from datasets import load_dataset\n",
    "dataset = load_dataset(\"oscar\", \"unshuffled_deduplicated_bn\", split=\"train[:1%]\")\n",
    "dataset = dataset.map(format_dataset, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c743ec2d-c0d9-49e9-b455-d09b2bf35c71",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth 2024.8: Fast Llama patching. Transformers = 4.44.0.dev0.\n",
      "   \\\\   /|    GPU: NVIDIA RTX A6000. Max memory: 47.529 GB. Platform = Linux.\n",
      "O^O/ \\_/ \\    Pytorch: 2.1.2+cu118. CUDA = 8.6. CUDA Toolkit = 11.8.\n",
      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.23.post1+cu118. FA2 = True]\n",
      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bea94e2d37474e759dac123e493dae35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/5.70G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3bf7ac31dc043c2aa7ff2a397bf853e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/184 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2eea304809ff405481b4d1711fae70eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/50.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39683198c1e5464cb1e7ec3a936a0b96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/9.09M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd2e66fb2f0d468f8c40b591dbcc9d6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/459 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize the model\n",
    "from unsloth import FastLanguageModel\n",
    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
    "    model_name=\"unsloth/Meta-Llama-3.1-8B\",\n",
    "    max_seq_length=2048,\n",
    "    dtype=None,\n",
    "    load_in_4bit=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "511436bf-4d47-484a-89ed-f382b5cb1e10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attach LoRA Adapters\n",
    "lora_config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=32,\n",
    "    target_modules=[\"q_proj\", \"v_proj\"],\n",
    "    lora_dropout=0.1,\n",
    "    bias=\"none\"\n",
    ")\n",
    "model = get_peft_model(model, lora_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "be47690d-67df-4f75-9f11-08bcffc64bdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define data collator\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=bangla_tokenizer,\n",
    "    mlm=False\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c5c818c8-83da-4730-a373-7fb032058d70",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./bangla_llama\",\n",
    "    overwrite_output_dir=True,\n",
    "    num_train_epochs=3,\n",
    "    per_device_train_batch_size=4,\n",
    "    save_steps=10_000,\n",
    "    save_total_limit=2,\n",
    "    logging_steps=1,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    eval_steps=10_000,\n",
    "    learning_rate=5e-5,\n",
    "    weight_decay=0.01,\n",
    "    report_to=\"wandb\",\n",
    "    remove_unused_columns=False\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f15dc8f6-fe00-4c4c-9686-884ce57a5171",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=dataset,\n",
    "    eval_dataset=dataset\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "24a7f7b1-416e-48a3-954b-c962f0095e24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:ranvymh0) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.012 MB of 0.012 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">misunderstood-meadow-13</strong> at: <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/ranvymh0/workspace' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/ranvymh0/workspace</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240806_153321-ranvymh0/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Successfully finished last run (ID:ranvymh0). Initializing new run:<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f7f3a29e1eb417f9f01621e058d4d8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011112969011689226, max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.17.5 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspace/wandb/run-20240806_154409-6rz00to1</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/6rz00to1/workspace' target=\"_blank\">icy-sponge-14</a></strong> to <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/6rz00to1/workspace' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/6rz00to1/workspace</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1\n",
      "   \\\\   /|    Num examples = 11,145 | Num Epochs = 3\n",
      "O^O/ \\_/ \\    Batch size per device = 4 | Gradient Accumulation steps = 1\n",
      "\\        /    Total batch size = 4 | Total steps = 8,361\n",
      " \"-____-\"     Number of trainable parameters = 6,815,744\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your features (`text` in this case) have excessive nesting (inputs type `list` where type `int` is expected).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:762\u001b[0m, in \u001b[0;36mBatchEncoding.convert_to_tensors\u001b[0;34m(self, tensor_type, prepend_batch_axis)\u001b[0m\n\u001b[1;32m    761\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_tensor(value):\n\u001b[0;32m--> 762\u001b[0m     tensor \u001b[38;5;241m=\u001b[39m \u001b[43mas_tensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    764\u001b[0m     \u001b[38;5;66;03m# Removing this for now in favor of controlling the shape with `prepend_batch_axis`\u001b[39;00m\n\u001b[1;32m    765\u001b[0m     \u001b[38;5;66;03m# # at-least2d\u001b[39;00m\n\u001b[1;32m    766\u001b[0m     \u001b[38;5;66;03m# if tensor.ndim > 2:\u001b[39;00m\n\u001b[1;32m    767\u001b[0m     \u001b[38;5;66;03m#     tensor = tensor.squeeze(0)\u001b[39;00m\n\u001b[1;32m    768\u001b[0m     \u001b[38;5;66;03m# elif tensor.ndim < 2:\u001b[39;00m\n\u001b[1;32m    769\u001b[0m     \u001b[38;5;66;03m#     tensor = tensor[None, :]\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:724\u001b[0m, in \u001b[0;36mBatchEncoding.convert_to_tensors.<locals>.as_tensor\u001b[0;34m(value, dtype)\u001b[0m\n\u001b[1;32m    723\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mtensor(np\u001b[38;5;241m.\u001b[39marray(value))\n\u001b[0;32m--> 724\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mValueError\u001b[0m: too many dimensions 'str'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[36], line 5\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(training_args\u001b[38;5;241m.\u001b[39mnum_train_epochs):\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEpoch \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepoch\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtraining_args\u001b[38;5;241m.\u001b[39mnum_train_epochs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 5\u001b[0m     \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# Save the model with progress bar\u001b[39;00m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSaving the model...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/trainer.py:1938\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1936\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1937\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1938\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1939\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1940\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1941\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1942\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1943\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m<string>:320\u001b[0m, in \u001b[0;36m_fast_inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/accelerate/data_loader.py:454\u001b[0m, in \u001b[0;36mDataLoaderShard.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[38;5;66;03m# We iterate one batch ahead to check when we are at the end\u001b[39;00m\n\u001b[1;32m    453\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 454\u001b[0m     current_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mdataloader_iter\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    455\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m    456\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/data/dataloader.py:630\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    627\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    628\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m    629\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 630\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    632\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    633\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    634\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/data/dataloader.py:674\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    673\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m--> 674\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m    675\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m    676\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py:54\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n\u001b[0;32m---> 54\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollate_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/data/data_collator.py:45\u001b[0m, in \u001b[0;36mDataCollatorMixin.__call__\u001b[0;34m(self, features, return_tensors)\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtf_call(features)\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m return_tensors \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m---> 45\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtorch_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     46\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m return_tensors \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnp\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnumpy_call(features)\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/data/data_collator.py:806\u001b[0m, in \u001b[0;36mDataCollatorForLanguageModeling.torch_call\u001b[0;34m(self, examples)\u001b[0m\n\u001b[1;32m    803\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtorch_call\u001b[39m(\u001b[38;5;28mself\u001b[39m, examples: List[Union[List[\u001b[38;5;28mint\u001b[39m], Any, Dict[\u001b[38;5;28mstr\u001b[39m, Any]]]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict[\u001b[38;5;28mstr\u001b[39m, Any]:\n\u001b[1;32m    804\u001b[0m     \u001b[38;5;66;03m# Handle dict or lists with proper padding and conversion to tensor.\u001b[39;00m\n\u001b[1;32m    805\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(examples[\u001b[38;5;241m0\u001b[39m], Mapping):\n\u001b[0;32m--> 806\u001b[0m         batch \u001b[38;5;241m=\u001b[39m \u001b[43mpad_without_fast_tokenizer_warning\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    807\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexamples\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_tensors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpt\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpad_to_multiple_of\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpad_to_multiple_of\u001b[49m\n\u001b[1;32m    808\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    809\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    810\u001b[0m         batch \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    811\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m: _torch_collate_batch(examples, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtokenizer, pad_to_multiple_of\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpad_to_multiple_of)\n\u001b[1;32m    812\u001b[0m         }\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/data/data_collator.py:66\u001b[0m, in \u001b[0;36mpad_without_fast_tokenizer_warning\u001b[0;34m(tokenizer, *pad_args, **pad_kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m tokenizer\u001b[38;5;241m.\u001b[39mdeprecation_warnings[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAsking-to-pad-a-fast-tokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 66\u001b[0m     padded \u001b[38;5;241m=\u001b[39m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpad\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mpad_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mpad_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     67\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# Restore the state of the warning.\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     tokenizer\u001b[38;5;241m.\u001b[39mdeprecation_warnings[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAsking-to-pad-a-fast-tokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m warning_state\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3546\u001b[0m, in \u001b[0;36mPreTrainedTokenizerBase.pad\u001b[0;34m(self, encoded_inputs, padding, max_length, pad_to_multiple_of, return_attention_mask, return_tensors, verbose)\u001b[0m\n\u001b[1;32m   3543\u001b[0m             batch_outputs[key] \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m   3544\u001b[0m         batch_outputs[key]\u001b[38;5;241m.\u001b[39mappend(value)\n\u001b[0;32m-> 3546\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mBatchEncoding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtensor_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_tensors\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:227\u001b[0m, in \u001b[0;36mBatchEncoding.__init__\u001b[0;34m(self, data, encoding, tensor_type, prepend_batch_axis, n_sequences)\u001b[0m\n\u001b[1;32m    223\u001b[0m     n_sequences \u001b[38;5;241m=\u001b[39m encoding[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mn_sequences\n\u001b[1;32m    225\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_n_sequences \u001b[38;5;241m=\u001b[39m n_sequences\n\u001b[0;32m--> 227\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconvert_to_tensors\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensor_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtensor_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprepend_batch_axis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprepend_batch_axis\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:778\u001b[0m, in \u001b[0;36mBatchEncoding.convert_to_tensors\u001b[0;34m(self, tensor_type, prepend_batch_axis)\u001b[0m\n\u001b[1;32m    773\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m key \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverflowing_tokens\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    774\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    775\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnable to create tensor returning overflowing tokens of different lengths. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    776\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease see if a fast version of this tokenizer is available to have this feature available.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    777\u001b[0m             ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[0;32m--> 778\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    779\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnable to create tensor, you should probably activate truncation and/or padding with\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    780\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpadding=True\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtruncation=True\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m to have batched tensors with the same length. Perhaps your\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    781\u001b[0m             \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m features (`\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` in this case) have excessive nesting (inputs type `list` where type `int` is\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    782\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m expected).\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    783\u001b[0m         ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m    785\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "\u001b[0;31mValueError\u001b[0m: Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your features (`text` in this case) have excessive nesting (inputs type `list` where type `int` is expected)."
     ]
    }
   ],
   "source": [
    "# Train the model with progress bar\n",
    "wandb.init(project=\"bangla-llama-finetuning\")\n",
    "for epoch in range(training_args.num_train_epochs):\n",
    "    print(f\"Epoch {epoch+1}/{training_args.num_train_epochs}\")\n",
    "    trainer.train()\n",
    "\n",
    "# Save the model with progress bar\n",
    "print(\"Saving the model...\")\n",
    "trainer.save_model(\"./bangla_llama\")\n",
    "bangla_tokenizer.save(\"bangla_llama\")\n",
    "print(\"Model saved.\")\n",
    "\n",
    "# Finish Weights & Biases run\n",
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01480966-690b-4ce7-ba54-6dc873bf4f35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
      "Token is valid (permission: fineGrained).\n",
      "Your token has been saved to /workspace/data/huggingface-cache/hub/token\n",
      "Login successful\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:6rz00to1) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.012 MB of 0.012 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">icy-sponge-14</strong> at: <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/6rz00to1/workspace' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/6rz00to1/workspace</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240806_154409-6rz00to1/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Successfully finished last run (ID:6rz00to1). Initializing new run:<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37fc4b2067fc4b6a971c36437e57ba8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011112884225116835, max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.17.5 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspace/wandb/run-20240806_154431-yu2wy6w5</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/yu2wy6w5/workspace' target=\"_blank\">fresh-silence-15</a></strong> to <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/yu2wy6w5/workspace' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/yu2wy6w5/workspace</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/datasets/load.py:1486: FutureWarning: The repository for oscar contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/oscar\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "925693b51bb340f28193e74cf2895f50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/11145 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth 2024.8: Fast Llama patching. Transformers = 4.44.0.dev0.\n",
      "   \\\\   /|    GPU: NVIDIA RTX A6000. Max memory: 47.529 GB. Platform = Linux.\n",
      "O^O/ \\_/ \\    Pytorch: 2.1.2+cu118. CUDA = 8.6. CUDA Toolkit = 11.8.\n",
      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.23.post1+cu118. FA2 = True]\n",
      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/py3.10/lib/python3.10/site-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:yu2wy6w5) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.012 MB of 0.012 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">fresh-silence-15</strong> at: <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/yu2wy6w5/workspace' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/yu2wy6w5/workspace</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240806_154431-yu2wy6w5/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Successfully finished last run (ID:yu2wy6w5). Initializing new run:<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d616e9f444324330bffc1d3588335d2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011113176365486449, max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.17.5 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspace/wandb/run-20240806_154757-hn6bdoy4</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/hn6bdoy4/workspace' target=\"_blank\">gentle-leaf-16</a></strong> to <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/hn6bdoy4/workspace' target=\"_blank\">https://wandb.ai/suvrakamal/bangla-llama-finetuning/runs/hn6bdoy4/workspace</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1\n",
      "   \\\\   /|    Num examples = 11,145 | Num Epochs = 3\n",
      "O^O/ \\_/ \\    Batch size per device = 4 | Gradient Accumulation steps = 1\n",
      "\\        /    Total batch size = 4 | Total steps = 8,361\n",
      " \"-____-\"     Number of trainable parameters = 6,815,744\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7178' max='8361' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7178/8361 1:31:45 < 15:07, 1.30 it/s, Epoch 2.58/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import wandb\n",
    "from datasets import load_dataset\n",
    "from tokenizers import Tokenizer\n",
    "from transformers import PreTrainedTokenizerFast, DataCollatorForLanguageModeling, Trainer, TrainingArguments\n",
    "import torch\n",
    "from unsloth import FastLanguageModel\n",
    "from tqdm import tqdm\n",
    "from huggingface_hub import login\n",
    "from peft import get_peft_model, LoraConfig\n",
    "\n",
    "# Login to Hugging Face\n",
    "login(\"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\")  # Replace with your Hugging Face token\n",
    "\n",
    "# Initialize Weights & Biases\n",
    "wandb.init(project=\"bangla-llama-finetuning\")\n",
    "\n",
    "# Load the custom Bangla tokenizer\n",
    "bangla_tokenizer = Tokenizer.from_file(\"bangla_tokenizer.json\")\n",
    "\n",
    "# Wrap the tokenizer\n",
    "bangla_tokenizer = PreTrainedTokenizerFast(tokenizer_object=bangla_tokenizer, unk_token=\"<unk>\", pad_token=\"<pad>\", cls_token=\"<s>\", sep_token=\"</s>\", mask_token=\"<mask>\")\n",
    "\n",
    "# Load the dataset\n",
    "dataset = load_dataset(\"oscar\", \"unshuffled_deduplicated_bn\", split=\"train[:1%]\")\n",
    "\n",
    "# Function to format the dataset for training\n",
    "def format_dataset(examples):\n",
    "    texts = examples[\"text\"]\n",
    "    # Tokenize the texts\n",
    "    tokenized_texts = bangla_tokenizer(texts, padding=\"max_length\", truncation=True, max_length=512)\n",
    "    return {\"input_ids\": tokenized_texts[\"input_ids\"], \"attention_mask\": tokenized_texts[\"attention_mask\"]}\n",
    "\n",
    "# Apply tokenization to the dataset\n",
    "dataset = dataset.map(format_dataset, batched=True)\n",
    "\n",
    "# Remove unnecessary columns\n",
    "dataset = dataset.remove_columns([\"id\", \"text\"])\n",
    "\n",
    "# Initialize the model\n",
    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
    "    model_name=\"unsloth/Meta-Llama-3.1-8B\",\n",
    "    max_seq_length=2048,\n",
    "    dtype=None,\n",
    "    load_in_4bit=True\n",
    ")\n",
    "\n",
    "# Attach LoRA Adapters\n",
    "lora_config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=32,\n",
    "    target_modules=[\"q_proj\", \"v_proj\"],\n",
    "    lora_dropout=0.1,\n",
    "    bias=\"none\"\n",
    ")\n",
    "model = get_peft_model(model, lora_config)\n",
    "\n",
    "# Define data collator\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=bangla_tokenizer,\n",
    "    mlm=False,\n",
    "    pad_to_multiple_of=8  # Optional: Ensure tensors are padded to be divisible by 8\n",
    ")\n",
    "\n",
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./bangla_llama\",\n",
    "    overwrite_output_dir=True,\n",
    "    num_train_epochs=3,\n",
    "    per_device_train_batch_size=4,\n",
    "    save_steps=10_000,\n",
    "    save_total_limit=2,\n",
    "    logging_steps=1,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    eval_steps=10_000,\n",
    "    learning_rate=5e-5,\n",
    "    weight_decay=0.01,\n",
    "    report_to=\"wandb\",\n",
    "    remove_unused_columns=False\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=dataset,\n",
    "    eval_dataset=dataset\n",
    ")\n",
    "\n",
    "# Train the model with progress bar\n",
    "wandb.init(project=\"bangla-llama-finetuning\")\n",
    "for epoch in range(training_args.num_train_epochs):\n",
    "    print(f\"Epoch {epoch+1}/{training_args.num_train_epochs}\")\n",
    "    trainer.train()\n",
    "\n",
    "# Save the model with progress bar\n",
    "print(\"Saving the model...\")\n",
    "trainer.save_model(\"./bangla_llama\")\n",
    "bangla_tokenizer.save(\"bangla_llama\")\n",
    "print(\"Model saved.\")\n",
    "\n",
    "# Finish Weights & Biases run\n",
    "wandb.finish()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96297d31-2ae9-4d58-865b-63c978bbf294",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}