Finetuning (Terminal).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aacc9986",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------------------- /opt/ohpc/pub/modulefiles ---------------------------\n",
      "   anaconda/2020.02    anaconda/2022.05\n",
      "   anaconda/2020.11    anaconda/2024.06 (D)\n",
      "\n",
      "  Where:\n",
      "   D:  Default Module\n",
      "\n",
      "Module defaults are chosen based on Find First Rules due to Name/Version/Version modules found in the module tree.\n",
      "See https://lmod.readthedocs.io/en/latest/060_locating.html for details.\n",
      "\n",
      "Use \"module spider\" to find all possible modules.\n",
      "Use \"module keyword key1 key2 ...\" to search for all possible modules matching\n",
      "any of the \"keys\".\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "module avail conda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06985cd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conda 24.5.0\n"
     ]
    }
   ],
   "source": [
    "module load anaconda/2024.06\n",
    "conda --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "07134759",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/u3/prasan\n"
     ]
    }
   ],
   "source": [
    "cd\n",
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "384a2584",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/u3/prasan/CameraTraps\n",
      "Dockerfile\t      PytorchWildlife  assets\t\trequirements.txt\n",
      "INSTALLATION.md       README.md        demo\t\troadmaps.md\n",
      "LICENSE\t\t      SECURITY.md      docs\t\tsetup.py\n",
      "PW_FT_classification  archive\t       megadetector.md\tversion.txt\n"
     ]
    }
   ],
   "source": [
    "cd CameraTraps\n",
    "pwd\n",
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cfe78779",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: torch in /home/u3/prasan/.local/lib/python3.12/site-packages (from -r requirements.txt (line 1)) (2.4.1)\n",
      "Requirement already satisfied: torchvision in /home/u3/prasan/.local/lib/python3.12/site-packages (from -r requirements.txt (line 2)) (0.19.1)\n",
      "Requirement already satisfied: torchaudio in /home/u3/prasan/.local/lib/python3.12/site-packages (from -r requirements.txt (line 3)) (2.4.1)\n",
      "Requirement already satisfied: tqdm in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from -r requirements.txt (line 4)) (4.66.4)\n",
      "Requirement already satisfied: Pillow in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from -r requirements.txt (line 5)) (10.3.0)\n",
      "Requirement already satisfied: supervision==0.19.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from -r requirements.txt (line 6)) (0.19.0)\n",
      "Requirement already satisfied: gradio in /home/u3/prasan/.local/lib/python3.12/site-packages (from -r requirements.txt (line 7)) (4.44.0)\n",
      "Requirement already satisfied: ultralytics-yolov5 in /home/u3/prasan/.local/lib/python3.12/site-packages (from -r requirements.txt (line 8)) (0.1.1)\n",
      "Requirement already satisfied: chardet in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from -r requirements.txt (line 9)) (4.0.0)\n",
      "Requirement already satisfied: defusedxml<0.8.0,>=0.7.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from supervision==0.19.0->-r requirements.txt (line 6)) (0.7.1)\n",
      "Requirement already satisfied: matplotlib>=3.6.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from supervision==0.19.0->-r requirements.txt (line 6)) (3.8.4)\n",
      "Requirement already satisfied: numpy>=1.21.2 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from supervision==0.19.0->-r requirements.txt (line 6)) (1.26.4)\n",
      "Requirement already satisfied: opencv-python-headless>=4.5.5.64 in /home/u3/prasan/.local/lib/python3.12/site-packages (from supervision==0.19.0->-r requirements.txt (line 6)) (4.10.0.84)\n",
      "Requirement already satisfied: pyyaml>=5.3 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from supervision==0.19.0->-r requirements.txt (line 6)) (6.0.1)\n",
      "Requirement already satisfied: scipy<2.0.0,>=1.10.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from supervision==0.19.0->-r requirements.txt (line 6)) (1.13.1)\n",
      "Requirement already satisfied: filelock in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (4.11.0)\n",
      "Requirement already satisfied: sympy in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (1.12)\n",
      "Requirement already satisfied: networkx in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (3.1.4)\n",
      "Requirement already satisfied: fsspec in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (2024.3.1)\n",
      "Requirement already satisfied: setuptools in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (69.5.1)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (9.1.0.70)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (2.20.5)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (12.1.105)\n",
      "Requirement already satisfied: triton==3.0.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from torch->-r requirements.txt (line 1)) (3.0.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/u3/prasan/.local/lib/python3.12/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->-r requirements.txt (line 1)) (12.6.68)\n",
      "Requirement already satisfied: aiofiles<24.0,>=22.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (23.2.1)\n",
      "Requirement already satisfied: anyio<5.0,>=3.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (4.2.0)\n",
      "Requirement already satisfied: fastapi<1.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.114.2)\n",
      "Requirement already satisfied: ffmpy in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.4.0)\n",
      "Requirement already satisfied: gradio-client==1.3.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (1.3.0)\n",
      "Requirement already satisfied: httpx>=0.24.1 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.27.2)\n",
      "Requirement already satisfied: huggingface-hub>=0.19.3 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.24.7)\n",
      "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (6.4.5)\n",
      "Requirement already satisfied: markupsafe~=2.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (2.1.3)\n",
      "Requirement already satisfied: orjson~=3.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (3.10.7)\n",
      "Requirement already satisfied: packaging in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (23.2)\n",
      "Requirement already satisfied: pandas<3.0,>=1.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (2.2.2)\n",
      "Requirement already satisfied: pydantic>=2.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (2.5.3)\n",
      "Requirement already satisfied: pydub in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.25.1)\n",
      "Requirement already satisfied: python-multipart>=0.0.9 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.0.9)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ruff>=0.2.2 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.6.5)\n",
      "Requirement already satisfied: semantic-version~=2.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (2.10.0)\n",
      "Requirement already satisfied: tomlkit==0.12.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.12.0)\n",
      "Requirement already satisfied: typer<1.0,>=0.12 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.12.5)\n",
      "Requirement already satisfied: urllib3~=2.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (2.2.2)\n",
      "Requirement already satisfied: uvicorn>=0.14.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio->-r requirements.txt (line 7)) (0.30.6)\n",
      "Requirement already satisfied: websockets<13.0,>=10.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from gradio-client==1.3.0->gradio->-r requirements.txt (line 7)) (12.0)\n",
      "Requirement already satisfied: opencv-python>=4.1.1 in /home/u3/prasan/.local/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (4.10.0.84)\n",
      "Requirement already satisfied: requests>=2.23.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (2.32.2)\n",
      "Requirement already satisfied: protobuf<=3.20.1 in /home/u3/prasan/.local/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (3.20.1)\n",
      "Requirement already satisfied: fire in /home/u3/prasan/.local/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (0.6.0)\n",
      "Requirement already satisfied: tensorboard>=2.4.1 in /home/u3/prasan/.local/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (2.17.1)\n",
      "Requirement already satisfied: seaborn>=0.11.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (0.13.2)\n",
      "Requirement already satisfied: ipython in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (8.25.0)\n",
      "Requirement already satisfied: psutil in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (5.9.0)\n",
      "Requirement already satisfied: thop in /home/u3/prasan/.local/lib/python3.12/site-packages (from ultralytics-yolov5->-r requirements.txt (line 8)) (0.1.1.post2209072238)\n",
      "Requirement already satisfied: idna>=2.8 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 7)) (3.7)\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from anyio<5.0,>=3.0->gradio->-r requirements.txt (line 7)) (1.3.0)\n",
      "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in /home/u3/prasan/.local/lib/python3.12/site-packages (from fastapi<1.0->gradio->-r requirements.txt (line 7)) (0.38.5)\n",
      "Requirement already satisfied: certifi in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 7)) (2024.6.2)\n",
      "Requirement already satisfied: httpcore==1.* in /home/u3/prasan/.local/lib/python3.12/site-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 7)) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/u3/prasan/.local/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.24.1->gradio->-r requirements.txt (line 7)) (0.14.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from matplotlib>=3.6.0->supervision==0.19.0->-r requirements.txt (line 6)) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from matplotlib>=3.6.0->supervision==0.19.0->-r requirements.txt (line 6)) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from matplotlib>=3.6.0->supervision==0.19.0->-r requirements.txt (line 6)) (4.51.0)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from matplotlib>=3.6.0->supervision==0.19.0->-r requirements.txt (line 6)) (1.4.4)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from matplotlib>=3.6.0->supervision==0.19.0->-r requirements.txt (line 6)) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from matplotlib>=3.6.0->supervision==0.19.0->-r requirements.txt (line 6)) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 7)) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from pandas<3.0,>=1.0->gradio->-r requirements.txt (line 7)) (2023.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from pydantic>=2.0->gradio->-r requirements.txt (line 7)) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.14.6 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from pydantic>=2.0->gradio->-r requirements.txt (line 7)) (2.14.6)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from requests>=2.23.0->ultralytics-yolov5->-r requirements.txt (line 8)) (2.0.4)\n",
      "Requirement already satisfied: absl-py>=0.4 in /home/u3/prasan/.local/lib/python3.12/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->-r requirements.txt (line 8)) (2.1.0)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in /home/u3/prasan/.local/lib/python3.12/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->-r requirements.txt (line 8)) (1.66.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->-r requirements.txt (line 8)) (3.4.1)\n",
      "Requirement already satisfied: six>1.9 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->-r requirements.txt (line 8)) (1.16.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->-r requirements.txt (line 8)) (0.7.2)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->-r requirements.txt (line 8)) (3.0.3)\n",
      "Requirement already satisfied: click>=8.0.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 7)) (8.1.7)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /home/u3/prasan/.local/lib/python3.12/site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 7)) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from typer<1.0,>=0.12->gradio->-r requirements.txt (line 7)) (13.3.5)\n",
      "Requirement already satisfied: termcolor in /home/u3/prasan/.local/lib/python3.12/site-packages (from fire->ultralytics-yolov5->-r requirements.txt (line 8)) (2.4.0)\n",
      "Requirement already satisfied: decorator in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.18.1)\n",
      "Requirement already satisfied: matplotlib-inline in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.1.6)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (3.0.43)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (2.15.1)\n",
      "Requirement already satisfied: stack-data in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.2.0)\n",
      "Requirement already satisfied: traitlets>=5.13.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (5.14.3)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (4.8.0)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from sympy->torch->-r requirements.txt (line 1)) (1.3.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from jedi>=0.16->ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.8.3)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from pexpect>4.3->ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.2.5)\n",
      "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio->-r requirements.txt (line 7)) (2.2.0)\n",
      "Requirement already satisfied: executing in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from stack-data->ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.8.3)\n",
      "Requirement already satisfied: asttokens in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from stack-data->ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (2.0.5)\n",
      "Requirement already satisfied: pure-eval in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from stack-data->ipython->ultralytics-yolov5->-r requirements.txt (line 8)) (0.2.2)\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/ohpc/pub/apps/anaconda/2024.06/lib/python3.12/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=10.11.0->typer<1.0,>=0.12->gradio->-r requirements.txt (line 7)) (0.1.0)\n"
     ]
    }
   ],
   "source": [
    "pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5d65e7c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/u3/prasan/CameraTraps/PW_FT_classification\n"
     ]
    }
   ],
   "source": [
    "cd PW_FT_classification\n",
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9c1312e",
   "metadata": {},
   "outputs": [],
   "source": [
    "conda env create -f environment.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf94c85d",
   "metadata": {},
   "outputs": [],
   "source": [
    "conda init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "601c70b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(PT_Finetuning) "
     ]
    },
    {
     "ename": "",
     "evalue": "1",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "conda activate PT_Finetuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1cfd9a2d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PytorchWildlife in /home/u3/prasan/.local/lib/python3.8/site-packages (from -r requirements.txt (line 1)) (1.0.2.17)\n",
      "Requirement already satisfied: scikit_learn in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from -r requirements.txt (line 2)) (1.2.0)\n",
      "Requirement already satisfied: lightning in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from -r requirements.txt (line 3)) (1.9.5)\n",
      "Requirement already satisfied: munch in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from -r requirements.txt (line 4)) (2.5.0)\n",
      "Requirement already satisfied: typer in /home/u3/prasan/.local/lib/python3.8/site-packages (from -r requirements.txt (line 5)) (0.12.5)\n",
      "Requirement already satisfied: numpy in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (1.24.4)\n",
      "Requirement already satisfied: torch==1.10.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (1.10.1)\n",
      "Requirement already satisfied: torchvision==0.11.2 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (0.11.2)\n",
      "Requirement already satisfied: torchaudio==0.10.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (0.10.1)\n",
      "Requirement already satisfied: tqdm==4.66.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (4.66.1)\n",
      "Requirement already satisfied: Pillow==10.1.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (10.1.0)\n",
      "Requirement already satisfied: supervision==0.19.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (0.19.0)\n",
      "Requirement already satisfied: gradio==4.8.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (4.8.0)\n",
      "Requirement already satisfied: ultralytics-yolov5 in /home/u3/prasan/.local/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (0.1.1)\n",
      "Requirement already satisfied: chardet in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from PytorchWildlife->-r requirements.txt (line 1)) (5.2.0)\n",
      "Requirement already satisfied: aiofiles<24.0,>=22.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (23.2.1)\n",
      "Requirement already satisfied: altair<6.0,>=4.2.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (5.4.1)\n",
      "Requirement already satisfied: fastapi in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.1.17)\n",
      "Requirement already satisfied: ffmpy in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.4.0)\n",
      "Requirement already satisfied: gradio-client==0.7.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.7.1)\n",
      "Requirement already satisfied: httpx in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.27.2)\n",
      "Requirement already satisfied: huggingface-hub>=0.14.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.25.0)\n",
      "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (6.1.1)\n",
      "Requirement already satisfied: jinja2<4.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.1.3)\n",
      "Requirement already satisfied: markupsafe~=2.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2.1.5)\n",
      "Requirement already satisfied: matplotlib~=3.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.7.5)\n",
      "Requirement already satisfied: orjson~=3.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.10.7)\n",
      "Requirement already satisfied: packaging in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (23.2)\n",
      "Requirement already satisfied: pandas<3.0,>=1.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2.0.3)\n",
      "Requirement already satisfied: pydantic>=2.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2.9.2)\n",
      "Requirement already satisfied: pydub in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.25.1)\n",
      "Requirement already satisfied: python-multipart in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.0.9)\n",
      "Requirement already satisfied: pyyaml<7.0,>=5.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (6.0.1)\n",
      "Requirement already satisfied: requests~=2.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2.31.0)\n",
      "Requirement already satisfied: semantic-version~=2.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2.10.0)\n",
      "Requirement already satisfied: tomlkit==0.12.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.12.0)\n",
      "Requirement already satisfied: typing-extensions~=4.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (4.12.2)\n",
      "Requirement already satisfied: uvicorn>=0.14.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.30.6)\n",
      "Requirement already satisfied: defusedxml<0.8.0,>=0.7.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from supervision==0.19.0->PytorchWildlife->-r requirements.txt (line 1)) (0.7.1)\n",
      "Requirement already satisfied: opencv-python-headless>=4.5.5.64 in /home/u3/prasan/.local/lib/python3.8/site-packages (from supervision==0.19.0->PytorchWildlife->-r requirements.txt (line 1)) (4.10.0.84)\n",
      "Requirement already satisfied: scipy==1.10.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from supervision==0.19.0->PytorchWildlife->-r requirements.txt (line 1)) (1.10.0)\n",
      "Requirement already satisfied: fsspec in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from gradio-client==0.7.1->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2023.12.2)\n",
      "Requirement already satisfied: websockets<12.0,>=10.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from gradio-client==0.7.1->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (11.0.3)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from scikit_learn->-r requirements.txt (line 2)) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from scikit_learn->-r requirements.txt (line 2)) (3.2.0)\n",
      "Requirement already satisfied: arrow<3.0,>=1.2.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (1.3.0)\n",
      "Requirement already satisfied: beautifulsoup4<6.0,>=4.8.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (4.12.3)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: click<10.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (8.1.7)\n",
      "Requirement already satisfied: croniter<1.4.0,>=1.3.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (1.3.15)\n",
      "Requirement already satisfied: dateutils<2.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (0.6.12)\n",
      "Requirement already satisfied: deepdiff<8.0,>=5.7.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (7.0.1)\n",
      "Requirement already satisfied: inquirer<5.0,>=2.10.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (3.4.0)\n",
      "Requirement already satisfied: lightning-cloud>=0.5.27 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (0.5.70)\n",
      "Requirement already satisfied: lightning-utilities<2.0,>=0.6.0.post0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (0.10.1)\n",
      "Requirement already satisfied: psutil<7.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (5.9.8)\n",
      "Requirement already satisfied: rich<15.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (13.8.1)\n",
      "Requirement already satisfied: starlette<2.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (0.38.5)\n",
      "Requirement already satisfied: starsessions<2.0,>=1.2.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (1.3.0)\n",
      "Requirement already satisfied: torchmetrics<2.0,>=0.7.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (1.4.2)\n",
      "Requirement already satisfied: traitlets<7.0,>=5.3.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (5.14.1)\n",
      "Requirement already satisfied: urllib3<3.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (1.26.20)\n",
      "Requirement already satisfied: websocket-client<3.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning->-r requirements.txt (line 3)) (1.8.0)\n",
      "Requirement already satisfied: six in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from munch->-r requirements.txt (line 4)) (1.16.0)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from typer->-r requirements.txt (line 5)) (1.5.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from arrow<3.0,>=1.2.0->lightning->-r requirements.txt (line 3)) (2.8.2)\n",
      "Requirement already satisfied: types-python-dateutil>=2.8.10 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from arrow<3.0,>=1.2.0->lightning->-r requirements.txt (line 3)) (2.9.0.20240906)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from beautifulsoup4<6.0,>=4.8.0->lightning->-r requirements.txt (line 3)) (2.6)\n",
      "Requirement already satisfied: pytz in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from dateutils<2.0->lightning->-r requirements.txt (line 3)) (2023.4)\n",
      "Requirement already satisfied: ordered-set<4.2.0,>=4.1.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from deepdiff<8.0,>=5.7.0->lightning->-r requirements.txt (line 3)) (4.1.0)\n",
      "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (3.10.5)\n",
      "Requirement already satisfied: blessed>=1.19.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from inquirer<5.0,>=2.10.0->lightning->-r requirements.txt (line 3)) (1.20.0)\n",
      "Requirement already satisfied: editor>=1.6.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from inquirer<5.0,>=2.10.0->lightning->-r requirements.txt (line 3)) (1.6.6)\n",
      "Requirement already satisfied: readchar>=4.2.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from inquirer<5.0,>=2.10.0->lightning->-r requirements.txt (line 3)) (4.2.0)\n",
      "Requirement already satisfied: pyjwt in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning-cloud>=0.5.27->lightning->-r requirements.txt (line 3)) (2.9.0)\n",
      "Requirement already satisfied: boto3 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning-cloud>=0.5.27->lightning->-r requirements.txt (line 3)) (1.35.19)\n",
      "Requirement already satisfied: protobuf in /home/u3/prasan/.local/lib/python3.8/site-packages (from lightning-cloud>=0.5.27->lightning->-r requirements.txt (line 3)) (3.20.1)\n",
      "Requirement already satisfied: setuptools in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from lightning-utilities<2.0,>=0.6.0.post0->lightning->-r requirements.txt (line 3)) (69.0.3)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from pydantic>=2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.23.4 in /home/u3/prasan/.local/lib/python3.8/site-packages (from pydantic>=2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2.23.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from requests~=2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from requests~=2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from requests~=2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2023.11.17)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from rich<15.0->lightning->-r requirements.txt (line 3)) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from rich<15.0->lightning->-r requirements.txt (line 3)) (2.18.0)\n",
      "Requirement already satisfied: anyio<5,>=3.4.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from starlette<2.0->lightning->-r requirements.txt (line 3)) (4.5.0)\n",
      "Requirement already satisfied: itsdangerous<3.0.0,>=2.0.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from starsessions<2.0,>=1.2.1->lightning->-r requirements.txt (line 3)) (2.2.0)\n",
      "Requirement already satisfied: h11>=0.8 in /home/u3/prasan/.local/lib/python3.8/site-packages (from uvicorn>=0.14.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.14.0)\n",
      "Requirement already satisfied: opencv-python>=4.1.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (4.10.0.84)\n",
      "Requirement already satisfied: fire in /home/u3/prasan/.local/lib/python3.8/site-packages (from ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.6.0)\n",
      "Requirement already satisfied: tensorboard>=2.4.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.14.0)\n",
      "Requirement already satisfied: seaborn>=0.11.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.13.2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ipython in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (8.12.3)\n",
      "Requirement already satisfied: thop in /home/u3/prasan/.local/lib/python3.8/site-packages (from ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.1.1.post2209072238)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (2.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/u3/prasan/.local/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/u3/prasan/.local/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (6.1.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (1.11.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<2025.0,>2021.06.0->lightning->-r requirements.txt (line 3)) (4.0.3)\n",
      "Requirement already satisfied: jsonschema>=3.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from altair<6.0,>=4.2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (4.21.1)\n",
      "Requirement already satisfied: narwhals>=1.5.2 in /home/u3/prasan/.local/lib/python3.8/site-packages (from altair<6.0,>=4.2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (1.8.1)\n",
      "Requirement already satisfied: sniffio>=1.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from anyio<5,>=3.4.0->starlette<2.0->lightning->-r requirements.txt (line 3)) (1.3.1)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/u3/prasan/.local/lib/python3.8/site-packages (from anyio<5,>=3.4.0->starlette<2.0->lightning->-r requirements.txt (line 3)) (1.2.2)\n",
      "Requirement already satisfied: wcwidth>=0.1.4 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from blessed>=1.19.0->inquirer<5.0,>=2.10.0->lightning->-r requirements.txt (line 3)) (0.2.13)\n",
      "Requirement already satisfied: runs in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from editor>=1.6.0->inquirer<5.0,>=2.10.0->lightning->-r requirements.txt (line 3)) (1.2.2)\n",
      "Requirement already satisfied: xmod in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from editor>=1.6.0->inquirer<5.0,>=2.10.0->lightning->-r requirements.txt (line 3)) (1.8.1)\n",
      "Requirement already satisfied: filelock in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from huggingface-hub>=0.14.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.13.1)\n",
      "Requirement already satisfied: zipp>=3.1.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from importlib-resources<7.0,>=1.3->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.17.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from markdown-it-py>=2.2.0->rich<15.0->lightning->-r requirements.txt (line 3)) (0.1.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from matplotlib~=3.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from matplotlib~=3.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from matplotlib~=3.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (4.47.2)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from matplotlib~=3.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (1.4.5)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from matplotlib~=3.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (3.1.1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from pandas<3.0,>=1.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2023.4)\n",
      "Requirement already satisfied: absl-py>=0.4 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.1.0)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (1.66.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.35.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (3.7)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.7.2)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (3.0.4)\n",
      "Requirement already satisfied: wheel>=0.26 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.42.0)\n",
      "\u001b[33mWARNING: typer 0.12.5 does not provide the extra 'all'\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: botocore<1.36.0,>=1.35.19 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from boto3->lightning-cloud>=0.5.27->lightning->-r requirements.txt (line 3)) (1.35.19)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from boto3->lightning-cloud>=0.5.27->lightning->-r requirements.txt (line 3)) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from boto3->lightning-cloud>=0.5.27->lightning->-r requirements.txt (line 3)) (0.10.2)\n",
      "Requirement already satisfied: termcolor in /home/u3/prasan/.local/lib/python3.8/site-packages (from fire->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.4.0)\n",
      "Requirement already satisfied: httpcore==1.* in /home/u3/prasan/.local/lib/python3.8/site-packages (from httpx->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (1.0.5)\n",
      "Requirement already satisfied: backcall in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.2.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: decorator in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.19.1)\n",
      "Requirement already satisfied: matplotlib-inline in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.1.6)\n",
      "Requirement already satisfied: pickleshare in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (3.0.43)\n",
      "Requirement already satisfied: stack-data in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.6.3)\n",
      "Requirement already satisfied: pexpect>4.3 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (4.9.0)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (5.5.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/u3/prasan/.local/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.4.1)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (4.9)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.0.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from jedi>=0.16->ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.8.3)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (2023.12.1)\n",
      "Requirement already satisfied: pkgutil-resolve-name>=1.3.10 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (1.3.10)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.33.0)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.8.0->PytorchWildlife->-r requirements.txt (line 1)) (0.17.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from markdown>=2.6.8->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (7.0.1)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from pexpect>4.3->ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.7.0)\n",
      "Requirement already satisfied: executing>=1.2.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from stack-data->ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.0.1)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from stack-data->ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (2.4.1)\n",
      "Requirement already satisfied: pure-eval in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from stack-data->ipython->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.2.2)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /home/u3/prasan/.conda/envs/PT_Finetuning/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (0.5.1)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /home/u3/prasan/.local/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.4.1->ultralytics-yolov5->PytorchWildlife->-r requirements.txt (line 1)) (3.2.2)\n",
      "(PT_Finetuning) "
     ]
    },
    {
     "ename": "",
     "evalue": "1",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a5955598",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Sep 26 08:28:57 2024       \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 550.90.07              Driver Version: 550.90.07      CUDA Version: 12.4     |\n",
      "|-----------------------------------------+------------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                        |               MIG M. |\n",
      "|=========================================+========================+======================|\n",
      "|   0  Tesla P100-PCIE-16GB           On  |   00000000:0B:00.0 Off |                    0 |\n",
      "| N/A   23C    P0             26W /  250W |       0MiB /  16384MiB |      0%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "|   1  Tesla P100-PCIE-16GB           On  |   00000000:86:00.0 Off |                    0 |\n",
      "| N/A   26C    P0             26W /  250W |       0MiB /  16384MiB |      0%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "                                                                                         \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                              |\n",
      "|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |\n",
      "|        ID   ID                                                               Usage      |\n",
      "|=========================================================================================|\n",
      "|  No running processes found                                                             |\n",
      "+-----------------------------------------------------------------------------------------+\n",
      "(PT_Finetuning) "
     ]
    },
    {
     "ename": "",
     "evalue": "1",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4513f9f3",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(PT_Finetuning) (PT_Finetuning) [rank: 0] Global seed set to 0\n",
      "Fusing layers... \n",
      "Fusing layers... \n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "100%|█████████████████████████████████████████| 237/237 [23:05<00:00,  5.85s/it]\n",
      "Fusing layers... \n",
      "Fusing layers... \n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "100%|█████████████████████████████████████████| 484/484 [44:03<00:00,  5.46s/it]\n",
      "Fusing layers... \n",
      "Fusing layers... \n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "100%|█████████████████████████████████████████| 570/570 [51:20<00:00,  5.40s/it]\n",
      "Number of images loaded:  7898\n",
      "Number of images loaded:  2769\n",
      "Number of images loaded:  2758\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name | Type                  | Params\n",
      "-----------------------------------------------\n",
      "0 | net  | PlainResNetClassifier | 25.6 M\n",
      "-----------------------------------------------\n",
      "25.6 M    Trainable params\n",
      "0         Non-trainable params\n",
      "25.6 M    Total params\n",
      "102.245   Total estimated model params size (MB)\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "Training: 0it [00:00, ?it/s]missing keys: ['bn1.num_batches_tracked', 'layer1.0.bn1.num_batches_tracked', 'layer1.0.bn2.num_batches_tracked', 'layer1.0.bn3.num_batches_tracked', 'layer1.0.downsample.1.num_batches_tracked', 'layer1.1.bn1.num_batches_tracked', 'layer1.1.bn2.num_batches_tracked', 'layer1.1.bn3.num_batches_tracked', 'layer1.2.bn1.num_batches_tracked', 'layer1.2.bn2.num_batches_tracked', 'layer1.2.bn3.num_batches_tracked', 'layer2.0.bn1.num_batches_tracked', 'layer2.0.bn2.num_batches_tracked', 'layer2.0.bn3.num_batches_tracked', 'layer2.0.downsample.1.num_batches_tracked', 'layer2.1.bn1.num_batches_tracked', 'layer2.1.bn2.num_batches_tracked', 'layer2.1.bn3.num_batches_tracked', 'layer2.2.bn1.num_batches_tracked', 'layer2.2.bn2.num_batches_tracked', 'layer2.2.bn3.num_batches_tracked', 'layer2.3.bn1.num_batches_tracked', 'layer2.3.bn2.num_batches_tracked', 'layer2.3.bn3.num_batches_tracked', 'layer3.0.bn1.num_batches_tracked', 'layer3.0.bn2.num_batches_tracked', 'layer3.0.bn3.num_batches_tracked', 'layer3.0.downsample.1.num_batches_tracked', 'layer3.1.bn1.num_batches_tracked', 'layer3.1.bn2.num_batches_tracked', 'layer3.1.bn3.num_batches_tracked', 'layer3.2.bn1.num_batches_tracked', 'layer3.2.bn2.num_batches_tracked', 'layer3.2.bn3.num_batches_tracked', 'layer3.3.bn1.num_batches_tracked', 'layer3.3.bn2.num_batches_tracked', 'layer3.3.bn3.num_batches_tracked', 'layer3.4.bn1.num_batches_tracked', 'layer3.4.bn2.num_batches_tracked', 'layer3.4.bn3.num_batches_tracked', 'layer3.5.bn1.num_batches_tracked', 'layer3.5.bn2.num_batches_tracked', 'layer3.5.bn3.num_batches_tracked', 'layer4.0.bn1.num_batches_tracked', 'layer4.0.bn2.num_batches_tracked', 'layer4.0.bn3.num_batches_tracked', 'layer4.0.downsample.1.num_batches_tracked', 'layer4.1.bn1.num_batches_tracked', 'layer4.1.bn2.num_batches_tracked', 'layer4.1.bn3.num_batches_tracked', 'layer4.2.bn1.num_batches_tracked', 'layer4.2.bn2.num_batches_tracked', 'layer4.2.bn3.num_batches_tracked']\n",
      "unused_keys: []\n",
      "Epoch 0:  74%|████████▊   | 247/334 [00:41<00:14,  5.96it/s, loss=0.82, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 0:  74%|████████▉   | 248/334 [00:41<00:14,  5.92it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  75%|████████▉   | 249/334 [00:41<00:14,  5.93it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  75%|████████▉   | 250/334 [00:42<00:14,  5.95it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  75%|█████████   | 251/334 [00:42<00:13,  5.97it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  75%|█████████   | 252/334 [00:42<00:13,  5.98it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  76%|█████████   | 253/334 [00:42<00:13,  6.00it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  76%|█████████▏  | 254/334 [00:42<00:13,  6.01it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  76%|█████████▏  | 255/334 [00:42<00:13,  6.03it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  77%|█████████▏  | 256/334 [00:42<00:12,  6.05it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  77%|█████████▏  | 257/334 [00:42<00:12,  6.06it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  77%|█████████▎  | 258/334 [00:42<00:12,  6.08it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  78%|█████████▎  | 259/334 [00:42<00:12,  6.10it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  78%|█████████▎  | 260/334 [00:42<00:12,  6.11it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  78%|█████████▍  | 261/334 [00:42<00:11,  6.13it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  78%|█████████▍  | 262/334 [00:42<00:11,  6.14it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  79%|█████████▍  | 263/334 [00:42<00:11,  6.16it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  79%|█████████▍  | 264/334 [00:42<00:11,  6.18it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  79%|█████████▌  | 265/334 [00:42<00:11,  6.19it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  80%|█████████▌  | 266/334 [00:42<00:10,  6.21it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  80%|█████████▌  | 267/334 [00:42<00:10,  6.22it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  80%|█████████▋  | 268/334 [00:42<00:10,  6.24it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  81%|█████████▋  | 269/334 [00:43<00:10,  6.25it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  81%|█████████▋  | 270/334 [00:43<00:10,  6.27it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  81%|█████████▋  | 271/334 [00:43<00:10,  6.29it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  81%|█████████▊  | 272/334 [00:43<00:09,  6.30it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  82%|█████████▊  | 273/334 [00:43<00:09,  6.32it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  82%|█████████▊  | 274/334 [00:43<00:09,  6.33it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  82%|█████████▉  | 275/334 [00:43<00:09,  6.35it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  83%|█████████▉  | 276/334 [00:43<00:09,  6.36it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  83%|█████████▉  | 277/334 [00:43<00:08,  6.38it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  83%|█████████▉  | 278/334 [00:43<00:08,  6.40it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  84%|██████████  | 279/334 [00:43<00:08,  6.41it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  84%|██████████  | 280/334 [00:43<00:08,  6.43it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  84%|██████████  | 281/334 [00:43<00:08,  6.44it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  84%|██████████▏ | 282/334 [00:43<00:08,  6.46it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  85%|██████████▏ | 283/334 [00:43<00:07,  6.47it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  85%|██████████▏ | 284/334 [00:43<00:07,  6.49it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  85%|██████████▏ | 285/334 [00:43<00:07,  6.50it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  86%|██████████▎ | 286/334 [00:43<00:07,  6.52it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  86%|██████████▎ | 287/334 [00:43<00:07,  6.53it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  86%|██████████▎ | 288/334 [00:43<00:07,  6.55it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  87%|██████████▍ | 289/334 [00:44<00:06,  6.56it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  87%|██████████▍ | 290/334 [00:44<00:06,  6.58it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  87%|██████████▍ | 291/334 [00:44<00:06,  6.59it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  87%|██████████▍ | 292/334 [00:44<00:06,  6.61it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  88%|██████████▌ | 293/334 [00:44<00:06,  6.62it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  88%|██████████▌ | 294/334 [00:44<00:06,  6.64it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  88%|██████████▌ | 295/334 [00:44<00:05,  6.65it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  89%|██████████▋ | 296/334 [00:44<00:05,  6.67it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  89%|██████████▋ | 297/334 [00:44<00:05,  6.68it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  89%|██████████▋ | 298/334 [00:44<00:05,  6.70it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  90%|██████████▋ | 299/334 [00:44<00:05,  6.71it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  90%|██████████▊ | 300/334 [00:44<00:05,  6.73it/s, loss=0.82, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:  90%|██████████▊ | 301/334 [00:44<00:04,  6.74it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  90%|██████████▊ | 302/334 [00:44<00:04,  6.76it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  91%|██████████▉ | 303/334 [00:44<00:04,  6.77it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  91%|██████████▉ | 304/334 [00:44<00:04,  6.79it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  91%|██████████▉ | 305/334 [00:44<00:04,  6.80it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  92%|██████████▉ | 306/334 [00:44<00:04,  6.82it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  92%|███████████ | 307/334 [00:44<00:03,  6.83it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  92%|███████████ | 308/334 [00:44<00:03,  6.85it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  93%|███████████ | 309/334 [00:45<00:03,  6.86it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  93%|███████████▏| 310/334 [00:45<00:03,  6.88it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  93%|███████████▏| 311/334 [00:45<00:03,  6.88it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  93%|███████████▏| 312/334 [00:45<00:03,  6.86it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  94%|███████████▏| 313/334 [00:45<00:03,  6.87it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  94%|███████████▎| 314/334 [00:45<00:02,  6.85it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  94%|███████████▎| 315/334 [00:45<00:02,  6.87it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  95%|███████████▎| 316/334 [00:45<00:02,  6.88it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  95%|███████████▍| 317/334 [00:45<00:02,  6.90it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  95%|███████████▍| 318/334 [00:46<00:02,  6.91it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  96%|███████████▍| 319/334 [00:46<00:02,  6.92it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  96%|███████████▍| 320/334 [00:46<00:02,  6.94it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  96%|███████████▌| 321/334 [00:46<00:01,  6.95it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  96%|███████████▌| 322/334 [00:46<00:01,  6.97it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  97%|███████████▌| 323/334 [00:46<00:01,  6.98it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  97%|███████████▋| 324/334 [00:46<00:01,  6.99it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  97%|███████████▋| 325/334 [00:46<00:01,  7.00it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  98%|███████████▋| 326/334 [00:46<00:01,  7.02it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  98%|███████████▋| 327/334 [00:46<00:00,  7.03it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  98%|███████████▊| 328/334 [00:46<00:00,  7.04it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  99%|███████████▊| 329/334 [00:46<00:00,  7.06it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  99%|███████████▊| 330/334 [00:46<00:00,  7.07it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  99%|███████████▉| 331/334 [00:46<00:00,  7.09it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0:  99%|███████████▉| 332/334 [00:46<00:00,  7.07it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0: 100%|███████████▉| 333/334 [00:46<00:00,  7.09it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0: 100%|████████████| 334/334 [00:47<00:00,  7.10it/s, loss=0.82, v_num=0]\u001b[A\n",
      "Epoch 0: 100%|████████████| 334/334 [00:47<00:00,  7.10it/s, loss=0.82, v_num=0]Epoch 0, global step 247: 'valid_mac_acc' reached 52.51141 (best 52.51141), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=00-valid_mac_acc=52.51.ckpt' as top 1\n",
      "Epoch 1:  74%|████████▏  | 247/334 [00:41<00:14,  5.97it/s, loss=0.489, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 1:  74%|████████▏  | 248/334 [00:41<00:14,  5.93it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  75%|████████▏  | 249/334 [00:41<00:14,  5.95it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  75%|████████▏  | 250/334 [00:41<00:14,  5.97it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  75%|████████▎  | 251/334 [00:41<00:13,  5.98it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  75%|████████▎  | 252/334 [00:42<00:13,  6.00it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  76%|████████▎  | 253/334 [00:42<00:13,  6.02it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  76%|████████▎  | 254/334 [00:42<00:13,  6.03it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  76%|████████▍  | 255/334 [00:42<00:13,  6.05it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  77%|████████▍  | 256/334 [00:42<00:12,  6.07it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  77%|████████▍  | 257/334 [00:42<00:12,  6.08it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  77%|████████▍  | 258/334 [00:42<00:12,  6.10it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  78%|████████▌  | 259/334 [00:42<00:12,  6.11it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  78%|████████▌  | 260/334 [00:42<00:12,  6.13it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  78%|████████▌  | 261/334 [00:42<00:11,  6.15it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  78%|████████▋  | 262/334 [00:42<00:11,  6.16it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  79%|████████▋  | 263/334 [00:42<00:11,  6.18it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  79%|████████▋  | 264/334 [00:42<00:11,  6.20it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  79%|████████▋  | 265/334 [00:42<00:11,  6.21it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  80%|████████▊  | 266/334 [00:42<00:10,  6.23it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  80%|████████▊  | 267/334 [00:42<00:10,  6.24it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  80%|████████▊  | 268/334 [00:42<00:10,  6.26it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  81%|████████▊  | 269/334 [00:42<00:10,  6.28it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  81%|████████▉  | 270/334 [00:42<00:10,  6.29it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  81%|████████▉  | 271/334 [00:42<00:09,  6.31it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  81%|████████▉  | 272/334 [00:43<00:09,  6.32it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  82%|████████▉  | 273/334 [00:43<00:09,  6.34it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  82%|█████████  | 274/334 [00:43<00:09,  6.35it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  82%|█████████  | 275/334 [00:43<00:09,  6.37it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  83%|█████████  | 276/334 [00:43<00:09,  6.39it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  83%|█████████  | 277/334 [00:43<00:08,  6.40it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  83%|█████████▏ | 278/334 [00:43<00:08,  6.42it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  84%|█████████▏ | 279/334 [00:43<00:08,  6.43it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  84%|█████████▏ | 280/334 [00:43<00:08,  6.45it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  84%|█████████▎ | 281/334 [00:43<00:08,  6.46it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  84%|█████████▎ | 282/334 [00:43<00:08,  6.48it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  85%|█████████▎ | 283/334 [00:43<00:07,  6.49it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  85%|█████████▎ | 284/334 [00:43<00:07,  6.51it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  85%|█████████▍ | 285/334 [00:43<00:07,  6.53it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  86%|█████████▍ | 286/334 [00:43<00:07,  6.54it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  86%|█████████▍ | 287/334 [00:43<00:07,  6.56it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  86%|█████████▍ | 288/334 [00:43<00:07,  6.57it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  87%|█████████▌ | 289/334 [00:43<00:06,  6.59it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  87%|█████████▌ | 290/334 [00:43<00:06,  6.60it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  87%|█████████▌ | 291/334 [00:43<00:06,  6.62it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  87%|█████████▌ | 292/334 [00:44<00:06,  6.63it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  88%|█████████▋ | 293/334 [00:44<00:06,  6.64it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  88%|█████████▋ | 294/334 [00:44<00:06,  6.66it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  88%|█████████▋ | 295/334 [00:44<00:05,  6.67it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  89%|█████████▋ | 296/334 [00:44<00:05,  6.69it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  89%|█████████▊ | 297/334 [00:44<00:05,  6.70it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  89%|█████████▊ | 298/334 [00:44<00:05,  6.72it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  90%|█████████▊ | 299/334 [00:44<00:05,  6.73it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  90%|█████████▉ | 300/334 [00:44<00:05,  6.75it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  90%|█████████▉ | 301/334 [00:44<00:04,  6.76it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  90%|█████████▉ | 302/334 [00:44<00:04,  6.78it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  91%|█████████▉ | 303/334 [00:44<00:04,  6.79it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  91%|██████████ | 304/334 [00:44<00:04,  6.81it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  91%|██████████ | 305/334 [00:44<00:04,  6.82it/s, loss=0.489, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1:  92%|██████████ | 306/334 [00:44<00:04,  6.84it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  92%|██████████ | 307/334 [00:44<00:03,  6.85it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  92%|██████████▏| 308/334 [00:44<00:03,  6.87it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  93%|██████████▏| 309/334 [00:44<00:03,  6.88it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  93%|██████████▏| 310/334 [00:44<00:03,  6.90it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  93%|██████████▏| 311/334 [00:45<00:03,  6.89it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  93%|██████████▎| 312/334 [00:45<00:03,  6.88it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  94%|██████████▎| 313/334 [00:45<00:03,  6.90it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  94%|██████████▎| 314/334 [00:45<00:02,  6.88it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  94%|██████████▎| 315/334 [00:45<00:02,  6.90it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  95%|██████████▍| 316/334 [00:45<00:02,  6.91it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  95%|██████████▍| 317/334 [00:45<00:02,  6.93it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  95%|██████████▍| 318/334 [00:45<00:02,  6.94it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  96%|██████████▌| 319/334 [00:45<00:02,  6.95it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  96%|██████████▌| 320/334 [00:45<00:02,  6.97it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  96%|██████████▌| 321/334 [00:45<00:01,  6.98it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  96%|██████████▌| 322/334 [00:46<00:01,  7.00it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  97%|██████████▋| 323/334 [00:46<00:01,  7.01it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  97%|██████████▋| 324/334 [00:46<00:01,  7.02it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  97%|██████████▋| 325/334 [00:46<00:01,  7.03it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  98%|██████████▋| 326/334 [00:46<00:01,  7.05it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  98%|██████████▊| 327/334 [00:46<00:00,  7.06it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  98%|██████████▊| 328/334 [00:46<00:00,  7.08it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  99%|██████████▊| 329/334 [00:46<00:00,  7.09it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  99%|██████████▊| 330/334 [00:46<00:00,  7.10it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  99%|██████████▉| 331/334 [00:46<00:00,  7.12it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1:  99%|██████████▉| 332/334 [00:46<00:00,  7.11it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1: 100%|██████████▉| 333/334 [00:46<00:00,  7.12it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1: 100%|███████████| 334/334 [00:46<00:00,  7.14it/s, loss=0.489, v_num=0]\u001b[A\n",
      "Epoch 1: 100%|███████████| 334/334 [00:46<00:00,  7.14it/s, loss=0.489, v_num=0]Epoch 1, global step 494: 'valid_mac_acc' reached 85.66748 (best 85.66748), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=01-valid_mac_acc=85.67.ckpt' as top 1\n",
      "Epoch 2:  74%|████████▏  | 247/334 [00:41<00:14,  5.97it/s, loss=0.458, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 2:  74%|████████▏  | 248/334 [00:41<00:14,  5.93it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  75%|████████▏  | 249/334 [00:41<00:14,  5.94it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  75%|████████▏  | 250/334 [00:41<00:14,  5.96it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  75%|████████▎  | 251/334 [00:42<00:13,  5.98it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  75%|████████▎  | 252/334 [00:42<00:13,  5.99it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  76%|████████▎  | 253/334 [00:42<00:13,  6.01it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  76%|████████▎  | 254/334 [00:42<00:13,  6.02it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  76%|████████▍  | 255/334 [00:42<00:13,  6.04it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  77%|████████▍  | 256/334 [00:42<00:12,  6.06it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  77%|████████▍  | 257/334 [00:42<00:12,  6.07it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  77%|████████▍  | 258/334 [00:42<00:12,  6.09it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  78%|████████▌  | 259/334 [00:42<00:12,  6.11it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  78%|████████▌  | 260/334 [00:42<00:12,  6.12it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  78%|████████▌  | 261/334 [00:42<00:11,  6.14it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  78%|████████▋  | 262/334 [00:42<00:11,  6.15it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  79%|████████▋  | 263/334 [00:42<00:11,  6.17it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  79%|████████▋  | 264/334 [00:42<00:11,  6.19it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  79%|████████▋  | 265/334 [00:42<00:11,  6.20it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  80%|████████▊  | 266/334 [00:42<00:10,  6.22it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  80%|████████▊  | 267/334 [00:42<00:10,  6.24it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  80%|████████▊  | 268/334 [00:42<00:10,  6.25it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  81%|████████▊  | 269/334 [00:42<00:10,  6.27it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  81%|████████▉  | 270/334 [00:42<00:10,  6.28it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  81%|████████▉  | 271/334 [00:43<00:10,  6.30it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  81%|████████▉  | 272/334 [00:43<00:09,  6.31it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  82%|████████▉  | 273/334 [00:43<00:09,  6.33it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  82%|█████████  | 274/334 [00:43<00:09,  6.34it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  82%|█████████  | 275/334 [00:43<00:09,  6.36it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  83%|█████████  | 276/334 [00:43<00:09,  6.38it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  83%|█████████  | 277/334 [00:43<00:08,  6.39it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  83%|█████████▏ | 278/334 [00:43<00:08,  6.41it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  84%|█████████▏ | 279/334 [00:43<00:08,  6.42it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  84%|█████████▏ | 280/334 [00:43<00:08,  6.44it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  84%|█████████▎ | 281/334 [00:43<00:08,  6.45it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  84%|█████████▎ | 282/334 [00:43<00:08,  6.47it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  85%|█████████▎ | 283/334 [00:43<00:07,  6.48it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  85%|█████████▎ | 284/334 [00:43<00:07,  6.50it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  85%|█████████▍ | 285/334 [00:43<00:07,  6.51it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  86%|█████████▍ | 286/334 [00:43<00:07,  6.53it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  86%|█████████▍ | 287/334 [00:43<00:07,  6.54it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  86%|█████████▍ | 288/334 [00:43<00:07,  6.56it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  87%|█████████▌ | 289/334 [00:43<00:06,  6.57it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  87%|█████████▌ | 290/334 [00:44<00:06,  6.59it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  87%|█████████▌ | 291/334 [00:44<00:06,  6.60it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  87%|█████████▌ | 292/334 [00:44<00:06,  6.62it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  88%|█████████▋ | 293/334 [00:44<00:06,  6.63it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  88%|█████████▋ | 294/334 [00:44<00:06,  6.65it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  88%|█████████▋ | 295/334 [00:44<00:05,  6.66it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  89%|█████████▋ | 296/334 [00:44<00:05,  6.68it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  89%|█████████▊ | 297/334 [00:44<00:05,  6.69it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  89%|█████████▊ | 298/334 [00:44<00:05,  6.71it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  90%|█████████▊ | 299/334 [00:44<00:05,  6.72it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  90%|█████████▉ | 300/334 [00:44<00:05,  6.74it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  90%|█████████▉ | 301/334 [00:44<00:04,  6.75it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  90%|█████████▉ | 302/334 [00:44<00:04,  6.77it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  91%|█████████▉ | 303/334 [00:44<00:04,  6.78it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  91%|██████████ | 304/334 [00:44<00:04,  6.80it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  91%|██████████ | 305/334 [00:44<00:04,  6.81it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  92%|██████████ | 306/334 [00:44<00:04,  6.83it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  92%|██████████ | 307/334 [00:44<00:03,  6.84it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  92%|██████████▏| 308/334 [00:44<00:03,  6.86it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  93%|██████████▏| 309/334 [00:44<00:03,  6.87it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  93%|██████████▏| 310/334 [00:45<00:03,  6.89it/s, loss=0.458, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2:  93%|██████████▏| 311/334 [00:45<00:03,  6.88it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  93%|██████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  94%|██████████▎| 313/334 [00:45<00:03,  6.89it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  94%|██████████▎| 314/334 [00:45<00:02,  6.87it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  94%|██████████▎| 315/334 [00:45<00:02,  6.88it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  95%|██████████▍| 316/334 [00:45<00:02,  6.90it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  95%|██████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  95%|██████████▍| 318/334 [00:45<00:02,  6.93it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  96%|██████████▌| 319/334 [00:45<00:02,  6.94it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  96%|██████████▌| 320/334 [00:46<00:02,  6.95it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  96%|██████████▌| 321/334 [00:46<00:01,  6.97it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  96%|██████████▌| 322/334 [00:46<00:01,  6.98it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  97%|██████████▋| 323/334 [00:46<00:01,  7.00it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  97%|██████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  97%|██████████▋| 325/334 [00:46<00:01,  7.01it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  98%|██████████▋| 326/334 [00:46<00:01,  7.03it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  98%|██████████▊| 327/334 [00:46<00:00,  7.04it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  98%|██████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  99%|██████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  99%|██████████▊| 330/334 [00:46<00:00,  7.08it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  99%|██████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2:  99%|██████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2: 100%|██████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.458, v_num=0]\u001b[A\n",
      "Epoch 2: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.458, v_num=0]Epoch 2, global step 741: 'valid_mac_acc' reached 89.07951 (best 89.07951), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=02-valid_mac_acc=89.08.ckpt' as top 1\n",
      "Epoch 3:  74%|████████▏  | 247/334 [00:41<00:14,  5.98it/s, loss=0.432, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 3:  74%|████████▏  | 248/334 [00:41<00:14,  5.94it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  75%|████████▏  | 249/334 [00:41<00:14,  5.95it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  75%|████████▏  | 250/334 [00:41<00:14,  5.97it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  75%|████████▎  | 251/334 [00:41<00:13,  5.99it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  75%|████████▎  | 252/334 [00:41<00:13,  6.00it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  76%|████████▎  | 253/334 [00:42<00:13,  6.02it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  76%|████████▎  | 254/334 [00:42<00:13,  6.04it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  76%|████████▍  | 255/334 [00:42<00:13,  6.05it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  77%|████████▍  | 256/334 [00:42<00:12,  6.07it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  77%|████████▍  | 257/334 [00:42<00:12,  6.08it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  77%|████████▍  | 258/334 [00:42<00:12,  6.10it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  78%|████████▌  | 259/334 [00:42<00:12,  6.12it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  78%|████████▌  | 260/334 [00:42<00:12,  6.13it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  78%|████████▌  | 261/334 [00:42<00:11,  6.15it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  78%|████████▋  | 262/334 [00:42<00:11,  6.16it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  79%|████████▋  | 263/334 [00:42<00:11,  6.18it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  79%|████████▋  | 264/334 [00:42<00:11,  6.20it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  79%|████████▋  | 265/334 [00:42<00:11,  6.21it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  80%|████████▊  | 266/334 [00:42<00:10,  6.23it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  80%|████████▊  | 267/334 [00:42<00:10,  6.24it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  80%|████████▊  | 268/334 [00:42<00:10,  6.26it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  81%|████████▊  | 269/334 [00:42<00:10,  6.28it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  81%|████████▉  | 270/334 [00:42<00:10,  6.29it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  81%|████████▉  | 271/334 [00:42<00:09,  6.31it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  81%|████████▉  | 272/334 [00:43<00:09,  6.32it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  82%|████████▉  | 273/334 [00:43<00:09,  6.34it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  82%|█████████  | 274/334 [00:43<00:09,  6.35it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  82%|█████████  | 275/334 [00:43<00:09,  6.37it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  83%|█████████  | 276/334 [00:43<00:09,  6.38it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  83%|█████████  | 277/334 [00:43<00:08,  6.40it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  83%|█████████▏ | 278/334 [00:43<00:08,  6.42it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  84%|█████████▏ | 279/334 [00:43<00:08,  6.43it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  84%|█████████▏ | 280/334 [00:43<00:08,  6.45it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  84%|█████████▎ | 281/334 [00:43<00:08,  6.46it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  84%|█████████▎ | 282/334 [00:43<00:08,  6.48it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  85%|█████████▎ | 283/334 [00:43<00:07,  6.49it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  85%|█████████▎ | 284/334 [00:43<00:07,  6.51it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  85%|█████████▍ | 285/334 [00:43<00:07,  6.52it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  86%|█████████▍ | 286/334 [00:43<00:07,  6.54it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  86%|█████████▍ | 287/334 [00:43<00:07,  6.55it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  86%|█████████▍ | 288/334 [00:43<00:07,  6.57it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  87%|█████████▌ | 289/334 [00:43<00:06,  6.58it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  87%|█████████▌ | 290/334 [00:43<00:06,  6.60it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  87%|█████████▌ | 291/334 [00:44<00:06,  6.61it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  87%|█████████▌ | 292/334 [00:44<00:06,  6.63it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  88%|█████████▋ | 293/334 [00:44<00:06,  6.64it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  88%|█████████▋ | 294/334 [00:44<00:06,  6.66it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  88%|█████████▋ | 295/334 [00:44<00:05,  6.67it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  89%|█████████▋ | 296/334 [00:44<00:05,  6.69it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  89%|█████████▊ | 297/334 [00:44<00:05,  6.70it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  89%|█████████▊ | 298/334 [00:44<00:05,  6.72it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  90%|█████████▊ | 299/334 [00:44<00:05,  6.73it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  90%|█████████▉ | 300/334 [00:44<00:05,  6.75it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  90%|█████████▉ | 301/334 [00:44<00:04,  6.76it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  90%|█████████▉ | 302/334 [00:44<00:04,  6.77it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  91%|█████████▉ | 303/334 [00:44<00:04,  6.79it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  91%|██████████ | 304/334 [00:44<00:04,  6.80it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  91%|██████████ | 305/334 [00:44<00:04,  6.82it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  92%|██████████ | 306/334 [00:44<00:04,  6.83it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  92%|██████████ | 307/334 [00:44<00:03,  6.85it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  92%|██████████▏| 308/334 [00:44<00:03,  6.86it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  93%|██████████▏| 309/334 [00:44<00:03,  6.88it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  93%|██████████▏| 310/334 [00:44<00:03,  6.89it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  93%|██████████▏| 311/334 [00:45<00:03,  6.89it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  93%|██████████▎| 312/334 [00:45<00:03,  6.88it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  94%|██████████▎| 313/334 [00:45<00:03,  6.89it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  94%|██████████▎| 314/334 [00:45<00:02,  6.87it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  94%|██████████▎| 315/334 [00:45<00:02,  6.89it/s, loss=0.432, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3:  95%|██████████▍| 316/334 [00:45<00:02,  6.90it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  95%|██████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  95%|██████████▍| 318/334 [00:45<00:02,  6.93it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  96%|██████████▌| 319/334 [00:45<00:02,  6.94it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  96%|██████████▌| 320/334 [00:45<00:02,  6.96it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  96%|██████████▌| 321/334 [00:46<00:01,  6.97it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  96%|██████████▌| 322/334 [00:46<00:01,  6.99it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  97%|██████████▋| 323/334 [00:46<00:01,  7.00it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  97%|██████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  97%|██████████▋| 325/334 [00:46<00:01,  7.02it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  98%|██████████▋| 326/334 [00:46<00:01,  7.03it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  98%|██████████▊| 327/334 [00:46<00:00,  7.04it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  98%|██████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  99%|██████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  99%|██████████▊| 330/334 [00:46<00:00,  7.09it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  99%|██████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3:  99%|██████████▉| 332/334 [00:46<00:00,  7.08it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3: 100%|██████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.432, v_num=0]\u001b[A\n",
      "Epoch 3: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.432, v_num=0]Epoch 3, global step 988: 'valid_mac_acc' was not in top 1\n",
      "Epoch 4:  74%|████████▏  | 247/334 [00:41<00:14,  5.97it/s, loss=0.401, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 4:  74%|████████▏  | 248/334 [00:41<00:14,  5.92it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  75%|████████▏  | 249/334 [00:41<00:14,  5.94it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  75%|████████▏  | 250/334 [00:41<00:14,  5.95it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  75%|████████▎  | 251/334 [00:42<00:13,  5.97it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  75%|████████▎  | 252/334 [00:42<00:13,  5.99it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  76%|████████▎  | 253/334 [00:42<00:13,  6.00it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  76%|████████▎  | 254/334 [00:42<00:13,  6.02it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  76%|████████▍  | 255/334 [00:42<00:13,  6.04it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  77%|████████▍  | 256/334 [00:42<00:12,  6.05it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  77%|████████▍  | 257/334 [00:42<00:12,  6.07it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  77%|████████▍  | 258/334 [00:42<00:12,  6.09it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  78%|████████▌  | 259/334 [00:42<00:12,  6.10it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  78%|████████▌  | 260/334 [00:42<00:12,  6.12it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  78%|████████▌  | 261/334 [00:42<00:11,  6.13it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  78%|████████▋  | 262/334 [00:42<00:11,  6.15it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  79%|████████▋  | 263/334 [00:42<00:11,  6.17it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  79%|████████▋  | 264/334 [00:42<00:11,  6.18it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  79%|████████▋  | 265/334 [00:42<00:11,  6.20it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  80%|████████▊  | 266/334 [00:42<00:10,  6.21it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  80%|████████▊  | 267/334 [00:42<00:10,  6.23it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  80%|████████▊  | 268/334 [00:42<00:10,  6.25it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  81%|████████▊  | 269/334 [00:42<00:10,  6.26it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  81%|████████▉  | 270/334 [00:43<00:10,  6.28it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  81%|████████▉  | 271/334 [00:43<00:10,  6.29it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  81%|████████▉  | 272/334 [00:43<00:09,  6.31it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  82%|████████▉  | 273/334 [00:43<00:09,  6.33it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  82%|█████████  | 274/334 [00:43<00:09,  6.34it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  82%|█████████  | 275/334 [00:43<00:09,  6.36it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  83%|█████████  | 276/334 [00:43<00:09,  6.37it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  83%|█████████  | 277/334 [00:43<00:08,  6.39it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  83%|█████████▏ | 278/334 [00:43<00:08,  6.40it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  84%|█████████▏ | 279/334 [00:43<00:08,  6.42it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  84%|█████████▏ | 280/334 [00:43<00:08,  6.43it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  84%|█████████▎ | 281/334 [00:43<00:08,  6.45it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  84%|█████████▎ | 282/334 [00:43<00:08,  6.46it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  85%|█████████▎ | 283/334 [00:43<00:07,  6.48it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  85%|█████████▎ | 284/334 [00:43<00:07,  6.49it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  85%|█████████▍ | 285/334 [00:43<00:07,  6.51it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  86%|█████████▍ | 286/334 [00:43<00:07,  6.52it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  86%|█████████▍ | 287/334 [00:43<00:07,  6.54it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  86%|█████████▍ | 288/334 [00:43<00:07,  6.55it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  87%|█████████▌ | 289/334 [00:43<00:06,  6.57it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  87%|█████████▌ | 290/334 [00:44<00:06,  6.58it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  87%|█████████▌ | 291/334 [00:44<00:06,  6.60it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  87%|█████████▌ | 292/334 [00:44<00:06,  6.61it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  88%|█████████▋ | 293/334 [00:44<00:06,  6.63it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  88%|█████████▋ | 294/334 [00:44<00:06,  6.64it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  88%|█████████▋ | 295/334 [00:44<00:05,  6.66it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  89%|█████████▋ | 296/334 [00:44<00:05,  6.67it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  89%|█████████▊ | 297/334 [00:44<00:05,  6.69it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  89%|█████████▊ | 298/334 [00:44<00:05,  6.70it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  90%|█████████▊ | 299/334 [00:44<00:05,  6.72it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  90%|█████████▉ | 300/334 [00:44<00:05,  6.73it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  90%|█████████▉ | 301/334 [00:44<00:04,  6.75it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  90%|█████████▉ | 302/334 [00:44<00:04,  6.76it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  91%|█████████▉ | 303/334 [00:44<00:04,  6.78it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  91%|██████████ | 304/334 [00:44<00:04,  6.79it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  91%|██████████ | 305/334 [00:44<00:04,  6.81it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  92%|██████████ | 306/334 [00:44<00:04,  6.82it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  92%|██████████ | 307/334 [00:44<00:03,  6.83it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  92%|██████████▏| 308/334 [00:44<00:03,  6.85it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  93%|██████████▏| 309/334 [00:45<00:03,  6.86it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  93%|██████████▏| 310/334 [00:45<00:03,  6.88it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  93%|██████████▏| 311/334 [00:45<00:03,  6.87it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  93%|██████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  94%|██████████▎| 313/334 [00:45<00:03,  6.88it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  94%|██████████▎| 314/334 [00:45<00:02,  6.86it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  94%|██████████▎| 315/334 [00:45<00:02,  6.87it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  95%|██████████▍| 316/334 [00:45<00:02,  6.89it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  95%|██████████▍| 317/334 [00:45<00:02,  6.90it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  95%|██████████▍| 318/334 [00:45<00:02,  6.91it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  96%|██████████▌| 319/334 [00:46<00:02,  6.93it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  96%|██████████▌| 320/334 [00:46<00:02,  6.94it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  96%|██████████▌| 321/334 [00:46<00:01,  6.96it/s, loss=0.401, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4:  96%|██████████▌| 322/334 [00:46<00:01,  6.97it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  97%|██████████▋| 323/334 [00:46<00:01,  6.98it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  97%|██████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  97%|██████████▋| 325/334 [00:46<00:01,  7.01it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  98%|██████████▋| 326/334 [00:46<00:01,  7.02it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  98%|██████████▊| 327/334 [00:46<00:00,  7.04it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  98%|██████████▊| 328/334 [00:46<00:00,  7.05it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  99%|██████████▊| 329/334 [00:46<00:00,  7.06it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  99%|██████████▊| 330/334 [00:46<00:00,  7.08it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  99%|██████████▉| 331/334 [00:46<00:00,  7.09it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4:  99%|██████████▉| 332/334 [00:46<00:00,  7.08it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4: 100%|██████████▉| 333/334 [00:46<00:00,  7.09it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.401, v_num=0]\u001b[A\n",
      "Epoch 4: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.401, v_num=0]Epoch 4, global step 1235: 'valid_mac_acc' reached 90.27382 (best 90.27382), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=04-valid_mac_acc=90.27.ckpt' as top 1\n",
      "Epoch 5:  74%|████████▏  | 247/334 [00:41<00:14,  5.98it/s, loss=0.413, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 5:  74%|████████▏  | 248/334 [00:41<00:14,  5.93it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  75%|████████▏  | 249/334 [00:41<00:14,  5.95it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  75%|████████▏  | 250/334 [00:41<00:14,  5.96it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  75%|████████▎  | 251/334 [00:41<00:13,  5.98it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  75%|████████▎  | 252/334 [00:42<00:13,  6.00it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  76%|████████▎  | 253/334 [00:42<00:13,  6.01it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  76%|████████▎  | 254/334 [00:42<00:13,  6.03it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  76%|████████▍  | 255/334 [00:42<00:13,  6.05it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  77%|████████▍  | 256/334 [00:42<00:12,  6.06it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  77%|████████▍  | 257/334 [00:42<00:12,  6.08it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  77%|████████▍  | 258/334 [00:42<00:12,  6.09it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  78%|████████▌  | 259/334 [00:42<00:12,  6.11it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  78%|████████▌  | 260/334 [00:42<00:12,  6.13it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  78%|████████▌  | 261/334 [00:42<00:11,  6.14it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  78%|████████▋  | 262/334 [00:42<00:11,  6.16it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  79%|████████▋  | 263/334 [00:42<00:11,  6.18it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  79%|████████▋  | 264/334 [00:42<00:11,  6.19it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  79%|████████▋  | 265/334 [00:42<00:11,  6.21it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  80%|████████▊  | 266/334 [00:42<00:10,  6.22it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  80%|████████▊  | 267/334 [00:42<00:10,  6.24it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  80%|████████▊  | 268/334 [00:42<00:10,  6.26it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  81%|████████▊  | 269/334 [00:42<00:10,  6.27it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  81%|████████▉  | 270/334 [00:42<00:10,  6.29it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  81%|████████▉  | 271/334 [00:42<00:09,  6.30it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  81%|████████▉  | 272/334 [00:43<00:09,  6.32it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  82%|████████▉  | 273/334 [00:43<00:09,  6.33it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  82%|█████████  | 274/334 [00:43<00:09,  6.35it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  82%|█████████  | 275/334 [00:43<00:09,  6.37it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  83%|█████████  | 276/334 [00:43<00:09,  6.38it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  83%|█████████  | 277/334 [00:43<00:08,  6.40it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  83%|█████████▏ | 278/334 [00:43<00:08,  6.41it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  84%|█████████▏ | 279/334 [00:43<00:08,  6.43it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  84%|█████████▏ | 280/334 [00:43<00:08,  6.44it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  84%|█████████▎ | 281/334 [00:43<00:08,  6.46it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  84%|█████████▎ | 282/334 [00:43<00:08,  6.48it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  85%|█████████▎ | 283/334 [00:43<00:07,  6.49it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  85%|█████████▎ | 284/334 [00:43<00:07,  6.51it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  85%|█████████▍ | 285/334 [00:43<00:07,  6.52it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  86%|█████████▍ | 286/334 [00:43<00:07,  6.54it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  86%|█████████▍ | 287/334 [00:43<00:07,  6.55it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  86%|█████████▍ | 288/334 [00:43<00:07,  6.57it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  87%|█████████▌ | 289/334 [00:43<00:06,  6.58it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  87%|█████████▌ | 290/334 [00:43<00:06,  6.60it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  87%|█████████▌ | 291/334 [00:44<00:06,  6.61it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  87%|█████████▌ | 292/334 [00:44<00:06,  6.63it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  88%|█████████▋ | 293/334 [00:44<00:06,  6.64it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  88%|█████████▋ | 294/334 [00:44<00:06,  6.66it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  88%|█████████▋ | 295/334 [00:44<00:05,  6.67it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  89%|█████████▋ | 296/334 [00:44<00:05,  6.69it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  89%|█████████▊ | 297/334 [00:44<00:05,  6.70it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  89%|█████████▊ | 298/334 [00:44<00:05,  6.72it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  90%|█████████▊ | 299/334 [00:44<00:05,  6.73it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  90%|█████████▉ | 300/334 [00:44<00:05,  6.75it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  90%|█████████▉ | 301/334 [00:44<00:04,  6.76it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  90%|█████████▉ | 302/334 [00:44<00:04,  6.78it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  91%|█████████▉ | 303/334 [00:44<00:04,  6.79it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  91%|██████████ | 304/334 [00:44<00:04,  6.81it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  91%|██████████ | 305/334 [00:44<00:04,  6.82it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  92%|██████████ | 306/334 [00:44<00:04,  6.84it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  92%|██████████ | 307/334 [00:44<00:03,  6.85it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  92%|██████████▏| 308/334 [00:44<00:03,  6.86it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  93%|██████████▏| 309/334 [00:44<00:03,  6.88it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  93%|██████████▏| 310/334 [00:44<00:03,  6.89it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  93%|██████████▏| 311/334 [00:45<00:03,  6.89it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  93%|██████████▎| 312/334 [00:45<00:03,  6.89it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  94%|██████████▎| 313/334 [00:45<00:03,  6.90it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  94%|██████████▎| 314/334 [00:45<00:02,  6.88it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  94%|██████████▎| 315/334 [00:45<00:02,  6.89it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  95%|██████████▍| 316/334 [00:45<00:02,  6.91it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  95%|██████████▍| 317/334 [00:45<00:02,  6.92it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  95%|██████████▍| 318/334 [00:45<00:02,  6.93it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  96%|██████████▌| 319/334 [00:45<00:02,  6.95it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  96%|██████████▌| 320/334 [00:45<00:02,  6.96it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  96%|██████████▌| 321/334 [00:46<00:01,  6.98it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  96%|██████████▌| 322/334 [00:46<00:01,  6.99it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  97%|██████████▋| 323/334 [00:46<00:01,  7.00it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  97%|██████████▋| 324/334 [00:46<00:01,  7.02it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  97%|██████████▋| 325/334 [00:46<00:01,  7.03it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  98%|██████████▋| 326/334 [00:46<00:01,  7.04it/s, loss=0.413, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5:  98%|██████████▊| 327/334 [00:46<00:00,  7.06it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  98%|██████████▊| 328/334 [00:46<00:00,  7.07it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  99%|██████████▊| 329/334 [00:46<00:00,  7.09it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  99%|██████████▊| 330/334 [00:46<00:00,  7.10it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  99%|██████████▉| 331/334 [00:46<00:00,  7.11it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5:  99%|██████████▉| 332/334 [00:46<00:00,  7.11it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5: 100%|██████████▉| 333/334 [00:46<00:00,  7.13it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5: 100%|███████████| 334/334 [00:46<00:00,  7.14it/s, loss=0.413, v_num=0]\u001b[A\n",
      "Epoch 5: 100%|███████████| 334/334 [00:46<00:00,  7.14it/s, loss=0.413, v_num=0]Epoch 5, global step 1482: 'valid_mac_acc' was not in top 1\n",
      "Epoch 6:  74%|█████████▌   | 247/334 [00:41<00:14,  5.97it/s, loss=0.4, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 6:  74%|█████████▋   | 248/334 [00:41<00:14,  5.93it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  75%|█████████▋   | 249/334 [00:41<00:14,  5.94it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  75%|█████████▋   | 250/334 [00:41<00:14,  5.96it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  75%|█████████▊   | 251/334 [00:41<00:13,  5.98it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  75%|█████████▊   | 252/334 [00:42<00:13,  5.99it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  76%|█████████▊   | 253/334 [00:42<00:13,  6.01it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  76%|█████████▉   | 254/334 [00:42<00:13,  6.03it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  76%|█████████▉   | 255/334 [00:42<00:13,  6.04it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  77%|█████████▉   | 256/334 [00:42<00:12,  6.06it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  77%|██████████   | 257/334 [00:42<00:12,  6.08it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  77%|██████████   | 258/334 [00:42<00:12,  6.09it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  78%|██████████   | 259/334 [00:42<00:12,  6.11it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  78%|██████████   | 260/334 [00:42<00:12,  6.12it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  78%|██████████▏  | 261/334 [00:42<00:11,  6.14it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  78%|██████████▏  | 262/334 [00:42<00:11,  6.16it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  79%|██████████▏  | 263/334 [00:42<00:11,  6.17it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  79%|██████████▎  | 264/334 [00:42<00:11,  6.19it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  79%|██████████▎  | 265/334 [00:42<00:11,  6.20it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  80%|██████████▎  | 266/334 [00:42<00:10,  6.22it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  80%|██████████▍  | 267/334 [00:42<00:10,  6.23it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  80%|██████████▍  | 268/334 [00:42<00:10,  6.25it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  81%|██████████▍  | 269/334 [00:42<00:10,  6.27it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  81%|██████████▌  | 270/334 [00:42<00:10,  6.28it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  81%|██████████▌  | 271/334 [00:43<00:10,  6.30it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  81%|██████████▌  | 272/334 [00:43<00:09,  6.31it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  82%|██████████▋  | 273/334 [00:43<00:09,  6.33it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  82%|██████████▋  | 274/334 [00:43<00:09,  6.34it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  82%|██████████▋  | 275/334 [00:43<00:09,  6.36it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  83%|██████████▋  | 276/334 [00:43<00:09,  6.37it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  83%|██████████▊  | 277/334 [00:43<00:08,  6.39it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  83%|██████████▊  | 278/334 [00:43<00:08,  6.40it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  84%|██████████▊  | 279/334 [00:43<00:08,  6.42it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  84%|██████████▉  | 280/334 [00:43<00:08,  6.43it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  84%|██████████▉  | 281/334 [00:43<00:08,  6.45it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  84%|██████████▉  | 282/334 [00:43<00:08,  6.47it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  85%|███████████  | 283/334 [00:43<00:07,  6.48it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  85%|███████████  | 284/334 [00:43<00:07,  6.50it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  85%|███████████  | 285/334 [00:43<00:07,  6.51it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  86%|███████████▏ | 286/334 [00:43<00:07,  6.53it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  86%|███████████▏ | 287/334 [00:43<00:07,  6.54it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  86%|███████████▏ | 288/334 [00:43<00:07,  6.56it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  87%|███████████▏ | 289/334 [00:43<00:06,  6.57it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  87%|███████████▎ | 290/334 [00:44<00:06,  6.59it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  87%|███████████▎ | 291/334 [00:44<00:06,  6.60it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  87%|███████████▎ | 292/334 [00:44<00:06,  6.62it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  88%|███████████▍ | 293/334 [00:44<00:06,  6.63it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  88%|███████████▍ | 294/334 [00:44<00:06,  6.65it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  88%|███████████▍ | 295/334 [00:44<00:05,  6.66it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  89%|███████████▌ | 296/334 [00:44<00:05,  6.68it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  89%|███████████▌ | 297/334 [00:44<00:05,  6.69it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  89%|███████████▌ | 298/334 [00:44<00:05,  6.71it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  90%|███████████▋ | 299/334 [00:44<00:05,  6.72it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  90%|███████████▋ | 300/334 [00:44<00:05,  6.73it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  90%|███████████▋ | 301/334 [00:44<00:04,  6.75it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  90%|███████████▊ | 302/334 [00:44<00:04,  6.76it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  91%|███████████▊ | 303/334 [00:44<00:04,  6.78it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  91%|███████████▊ | 304/334 [00:44<00:04,  6.79it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  91%|███████████▊ | 305/334 [00:44<00:04,  6.81it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  92%|███████████▉ | 306/334 [00:44<00:04,  6.82it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  92%|███████████▉ | 307/334 [00:44<00:03,  6.84it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  92%|███████████▉ | 308/334 [00:44<00:03,  6.85it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  93%|████████████ | 309/334 [00:45<00:03,  6.87it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  93%|████████████ | 310/334 [00:45<00:03,  6.88it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  93%|████████████ | 311/334 [00:45<00:03,  6.88it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  93%|████████████▏| 312/334 [00:45<00:03,  6.87it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  94%|████████████▏| 313/334 [00:45<00:03,  6.89it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  94%|████████████▏| 314/334 [00:45<00:02,  6.87it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  94%|████████████▎| 315/334 [00:45<00:02,  6.88it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  95%|████████████▎| 316/334 [00:45<00:02,  6.90it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  95%|████████████▎| 317/334 [00:45<00:02,  6.91it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  95%|████████████▍| 318/334 [00:45<00:02,  6.92it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  96%|████████████▍| 319/334 [00:45<00:02,  6.94it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  96%|████████████▍| 320/334 [00:46<00:02,  6.95it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  96%|████████████▍| 321/334 [00:46<00:01,  6.97it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  96%|████████████▌| 322/334 [00:46<00:01,  6.98it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  97%|████████████▌| 323/334 [00:46<00:01,  6.99it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  97%|████████████▌| 324/334 [00:46<00:01,  7.01it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  97%|████████████▋| 325/334 [00:46<00:01,  7.02it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  98%|████████████▋| 326/334 [00:46<00:01,  7.03it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  98%|████████████▋| 327/334 [00:46<00:00,  7.05it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  98%|████████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  99%|████████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  99%|████████████▊| 330/334 [00:46<00:00,  7.09it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  99%|████████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6:  99%|████████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.4, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|████████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6: 100%|█████████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.4, v_num=0]\u001b[A\n",
      "Epoch 6: 100%|█████████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.4, v_num=0]Epoch 6, global step 1729: 'valid_mac_acc' was not in top 1\n",
      "Epoch 7:  74%|████████▏  | 247/334 [00:41<00:14,  5.95it/s, loss=0.393, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 7:  74%|████████▏  | 248/334 [00:41<00:14,  5.91it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  75%|████████▏  | 249/334 [00:41<00:14,  5.93it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  75%|████████▏  | 250/334 [00:42<00:14,  5.95it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  75%|████████▎  | 251/334 [00:42<00:13,  5.96it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  75%|████████▎  | 252/334 [00:42<00:13,  5.98it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  76%|████████▎  | 253/334 [00:42<00:13,  6.00it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  76%|████████▎  | 254/334 [00:42<00:13,  6.01it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  76%|████████▍  | 255/334 [00:42<00:13,  6.03it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  77%|████████▍  | 256/334 [00:42<00:12,  6.05it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  77%|████████▍  | 257/334 [00:42<00:12,  6.06it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  77%|████████▍  | 258/334 [00:42<00:12,  6.08it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  78%|████████▌  | 259/334 [00:42<00:12,  6.09it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  78%|████████▌  | 260/334 [00:42<00:12,  6.11it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  78%|████████▌  | 261/334 [00:42<00:11,  6.13it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  78%|████████▋  | 262/334 [00:42<00:11,  6.14it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  79%|████████▋  | 263/334 [00:42<00:11,  6.16it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  79%|████████▋  | 264/334 [00:42<00:11,  6.18it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  79%|████████▋  | 265/334 [00:42<00:11,  6.19it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  80%|████████▊  | 266/334 [00:42<00:10,  6.21it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  80%|████████▊  | 267/334 [00:42<00:10,  6.22it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  80%|████████▊  | 268/334 [00:42<00:10,  6.24it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  81%|████████▊  | 269/334 [00:43<00:10,  6.26it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  81%|████████▉  | 270/334 [00:43<00:10,  6.27it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  81%|████████▉  | 271/334 [00:43<00:10,  6.29it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  81%|████████▉  | 272/334 [00:43<00:09,  6.30it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  82%|████████▉  | 273/334 [00:43<00:09,  6.32it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  82%|█████████  | 274/334 [00:43<00:09,  6.33it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  82%|█████████  | 275/334 [00:43<00:09,  6.35it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  83%|█████████  | 276/334 [00:43<00:09,  6.37it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  83%|█████████  | 277/334 [00:43<00:08,  6.38it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  83%|█████████▏ | 278/334 [00:43<00:08,  6.40it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  84%|█████████▏ | 279/334 [00:43<00:08,  6.41it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  84%|█████████▏ | 280/334 [00:43<00:08,  6.43it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  84%|█████████▎ | 281/334 [00:43<00:08,  6.44it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  84%|█████████▎ | 282/334 [00:43<00:08,  6.46it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  85%|█████████▎ | 283/334 [00:43<00:07,  6.47it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  85%|█████████▎ | 284/334 [00:43<00:07,  6.49it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  85%|█████████▍ | 285/334 [00:43<00:07,  6.50it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  86%|█████████▍ | 286/334 [00:43<00:07,  6.52it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  86%|█████████▍ | 287/334 [00:43<00:07,  6.53it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  86%|█████████▍ | 288/334 [00:43<00:07,  6.55it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  87%|█████████▌ | 289/334 [00:44<00:06,  6.56it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  87%|█████████▌ | 290/334 [00:44<00:06,  6.58it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  87%|█████████▌ | 291/334 [00:44<00:06,  6.60it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  87%|█████████▌ | 292/334 [00:44<00:06,  6.61it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  88%|█████████▋ | 293/334 [00:44<00:06,  6.63it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  88%|█████████▋ | 294/334 [00:44<00:06,  6.64it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  88%|█████████▋ | 295/334 [00:44<00:05,  6.66it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  89%|█████████▋ | 296/334 [00:44<00:05,  6.67it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  89%|█████████▊ | 297/334 [00:44<00:05,  6.69it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  89%|█████████▊ | 298/334 [00:44<00:05,  6.70it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  90%|█████████▊ | 299/334 [00:44<00:05,  6.71it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  90%|█████████▉ | 300/334 [00:44<00:05,  6.73it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  90%|█████████▉ | 301/334 [00:44<00:04,  6.74it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  90%|█████████▉ | 302/334 [00:44<00:04,  6.76it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  91%|█████████▉ | 303/334 [00:44<00:04,  6.77it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  91%|██████████ | 304/334 [00:44<00:04,  6.79it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  91%|██████████ | 305/334 [00:44<00:04,  6.80it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  92%|██████████ | 306/334 [00:44<00:04,  6.82it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  92%|██████████ | 307/334 [00:44<00:03,  6.83it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  92%|██████████▏| 308/334 [00:44<00:03,  6.85it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  93%|██████████▏| 309/334 [00:45<00:03,  6.86it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  93%|██████████▏| 310/334 [00:45<00:03,  6.88it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  93%|██████████▏| 311/334 [00:45<00:03,  6.87it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  93%|██████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  94%|██████████▎| 313/334 [00:45<00:03,  6.88it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  94%|██████████▎| 314/334 [00:45<00:02,  6.86it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  94%|██████████▎| 315/334 [00:45<00:02,  6.88it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  95%|██████████▍| 316/334 [00:45<00:02,  6.89it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  95%|██████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  95%|██████████▍| 318/334 [00:45<00:02,  6.92it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  96%|██████████▌| 319/334 [00:46<00:02,  6.93it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  96%|██████████▌| 320/334 [00:46<00:02,  6.95it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  96%|██████████▌| 321/334 [00:46<00:01,  6.96it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  96%|██████████▌| 322/334 [00:46<00:01,  6.98it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  97%|██████████▋| 323/334 [00:46<00:01,  6.99it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  97%|██████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  97%|██████████▋| 325/334 [00:46<00:01,  7.01it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  98%|██████████▋| 326/334 [00:46<00:01,  7.02it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  98%|██████████▊| 327/334 [00:46<00:00,  7.04it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  98%|██████████▊| 328/334 [00:46<00:00,  7.05it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  99%|██████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  99%|██████████▊| 330/334 [00:46<00:00,  7.08it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  99%|██████████▉| 331/334 [00:46<00:00,  7.09it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7:  99%|██████████▉| 332/334 [00:46<00:00,  7.08it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7: 100%|██████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.393, v_num=0]\u001b[A\n",
      "Epoch 7: 100%|███████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.393, v_num=0]Epoch 7, global step 1976: 'valid_mac_acc' was not in top 1\n",
      "Epoch 8:  74%|████████▏  | 247/334 [00:41<00:14,  5.97it/s, loss=0.356, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 8:  74%|████████▏  | 248/334 [00:41<00:14,  5.93it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  75%|████████▏  | 249/334 [00:41<00:14,  5.94it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  75%|████████▏  | 250/334 [00:41<00:14,  5.96it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  75%|████████▎  | 251/334 [00:42<00:13,  5.98it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  75%|████████▎  | 252/334 [00:42<00:13,  5.99it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  76%|████████▎  | 253/334 [00:42<00:13,  6.01it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  76%|████████▎  | 254/334 [00:42<00:13,  6.03it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  76%|████████▍  | 255/334 [00:42<00:13,  6.04it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  77%|████████▍  | 256/334 [00:42<00:12,  6.06it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  77%|████████▍  | 257/334 [00:42<00:12,  6.07it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  77%|████████▍  | 258/334 [00:42<00:12,  6.09it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  78%|████████▌  | 259/334 [00:42<00:12,  6.11it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  78%|████████▌  | 260/334 [00:42<00:12,  6.12it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  78%|████████▌  | 261/334 [00:42<00:11,  6.14it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  78%|████████▋  | 262/334 [00:42<00:11,  6.15it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  79%|████████▋  | 263/334 [00:42<00:11,  6.17it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  79%|████████▋  | 264/334 [00:42<00:11,  6.19it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  79%|████████▋  | 265/334 [00:42<00:11,  6.20it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  80%|████████▊  | 266/334 [00:42<00:10,  6.22it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  80%|████████▊  | 267/334 [00:42<00:10,  6.23it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  80%|████████▊  | 268/334 [00:42<00:10,  6.25it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  81%|████████▊  | 269/334 [00:42<00:10,  6.27it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  81%|████████▉  | 270/334 [00:42<00:10,  6.28it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  81%|████████▉  | 271/334 [00:43<00:10,  6.30it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  81%|████████▉  | 272/334 [00:43<00:09,  6.31it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  82%|████████▉  | 273/334 [00:43<00:09,  6.33it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  82%|█████████  | 274/334 [00:43<00:09,  6.34it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  82%|█████████  | 275/334 [00:43<00:09,  6.36it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  83%|█████████  | 276/334 [00:43<00:09,  6.38it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  83%|█████████  | 277/334 [00:43<00:08,  6.39it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  83%|█████████▏ | 278/334 [00:43<00:08,  6.41it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  84%|█████████▏ | 279/334 [00:43<00:08,  6.42it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  84%|█████████▏ | 280/334 [00:43<00:08,  6.44it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  84%|█████████▎ | 281/334 [00:43<00:08,  6.45it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  84%|█████████▎ | 282/334 [00:43<00:08,  6.47it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  85%|█████████▎ | 283/334 [00:43<00:07,  6.48it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  85%|█████████▎ | 284/334 [00:43<00:07,  6.50it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  85%|█████████▍ | 285/334 [00:43<00:07,  6.51it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  86%|█████████▍ | 286/334 [00:43<00:07,  6.53it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  86%|█████████▍ | 287/334 [00:43<00:07,  6.54it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  86%|█████████▍ | 288/334 [00:43<00:07,  6.56it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  87%|█████████▌ | 289/334 [00:43<00:06,  6.57it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  87%|█████████▌ | 290/334 [00:44<00:06,  6.59it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  87%|█████████▌ | 291/334 [00:44<00:06,  6.60it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  87%|█████████▌ | 292/334 [00:44<00:06,  6.62it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  88%|█████████▋ | 293/334 [00:44<00:06,  6.63it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  88%|█████████▋ | 294/334 [00:44<00:06,  6.65it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  88%|█████████▋ | 295/334 [00:44<00:05,  6.66it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  89%|█████████▋ | 296/334 [00:44<00:05,  6.68it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  89%|█████████▊ | 297/334 [00:44<00:05,  6.69it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  89%|█████████▊ | 298/334 [00:44<00:05,  6.71it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  90%|█████████▊ | 299/334 [00:44<00:05,  6.72it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  90%|█████████▉ | 300/334 [00:44<00:05,  6.74it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  90%|█████████▉ | 301/334 [00:44<00:04,  6.75it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  90%|█████████▉ | 302/334 [00:44<00:04,  6.77it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  91%|█████████▉ | 303/334 [00:44<00:04,  6.78it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  91%|██████████ | 304/334 [00:44<00:04,  6.80it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  91%|██████████ | 305/334 [00:44<00:04,  6.81it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  92%|██████████ | 306/334 [00:44<00:04,  6.82it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  92%|██████████ | 307/334 [00:44<00:03,  6.84it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  92%|██████████▏| 308/334 [00:44<00:03,  6.85it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  93%|██████████▏| 309/334 [00:44<00:03,  6.87it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  93%|██████████▏| 310/334 [00:45<00:03,  6.88it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  93%|██████████▏| 311/334 [00:45<00:03,  6.88it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  93%|██████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  94%|██████████▎| 313/334 [00:45<00:03,  6.89it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  94%|██████████▎| 314/334 [00:45<00:02,  6.87it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  94%|██████████▎| 315/334 [00:45<00:02,  6.88it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  95%|██████████▍| 316/334 [00:45<00:02,  6.90it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  95%|██████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  95%|██████████▍| 318/334 [00:45<00:02,  6.93it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  96%|██████████▌| 319/334 [00:45<00:02,  6.94it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  96%|██████████▌| 320/334 [00:46<00:02,  6.95it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  96%|██████████▌| 321/334 [00:46<00:01,  6.97it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  96%|██████████▌| 322/334 [00:46<00:01,  6.98it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  97%|██████████▋| 323/334 [00:46<00:01,  7.00it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  97%|██████████▋| 324/334 [00:46<00:01,  7.01it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  97%|██████████▋| 325/334 [00:46<00:01,  7.02it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  98%|██████████▋| 326/334 [00:46<00:01,  7.04it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  98%|██████████▊| 327/334 [00:46<00:00,  7.05it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  98%|██████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  99%|██████████▊| 329/334 [00:46<00:00,  7.08it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  99%|██████████▊| 330/334 [00:46<00:00,  7.09it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  99%|██████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8:  99%|██████████▉| 332/334 [00:46<00:00,  7.10it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8: 100%|██████████▉| 333/334 [00:46<00:00,  7.11it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8: 100%|███████████| 334/334 [00:46<00:00,  7.13it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 8: 100%|███████████| 334/334 [00:46<00:00,  7.13it/s, loss=0.356, v_num=0]Epoch 8, global step 2223: 'valid_mac_acc' reached 90.50271 (best 90.50271), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=08-valid_mac_acc=90.50.ckpt' as top 1\n",
      "Epoch 9:  74%|████████▏  | 247/334 [00:41<00:14,  5.99it/s, loss=0.414, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 9:  74%|████████▏  | 248/334 [00:41<00:14,  5.94it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  75%|████████▏  | 249/334 [00:41<00:14,  5.96it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  75%|████████▏  | 250/334 [00:41<00:14,  5.98it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  75%|████████▎  | 251/334 [00:41<00:13,  5.99it/s, loss=0.414, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9:  75%|████████▎  | 252/334 [00:41<00:13,  6.01it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  76%|████████▎  | 253/334 [00:41<00:13,  6.03it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  76%|████████▎  | 254/334 [00:42<00:13,  6.04it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  76%|████████▍  | 255/334 [00:42<00:13,  6.06it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  77%|████████▍  | 256/334 [00:42<00:12,  6.07it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  77%|████████▍  | 257/334 [00:42<00:12,  6.09it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  77%|████████▍  | 258/334 [00:42<00:12,  6.11it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  78%|████████▌  | 259/334 [00:42<00:12,  6.12it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  78%|████████▌  | 260/334 [00:42<00:12,  6.14it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  78%|████████▌  | 261/334 [00:42<00:11,  6.16it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  78%|████████▋  | 262/334 [00:42<00:11,  6.17it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  79%|████████▋  | 263/334 [00:42<00:11,  6.19it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  79%|████████▋  | 264/334 [00:42<00:11,  6.20it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  79%|████████▋  | 265/334 [00:42<00:11,  6.22it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  80%|████████▊  | 266/334 [00:42<00:10,  6.24it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  80%|████████▊  | 267/334 [00:42<00:10,  6.25it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  80%|████████▊  | 268/334 [00:42<00:10,  6.27it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  81%|████████▊  | 269/334 [00:42<00:10,  6.28it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  81%|████████▉  | 270/334 [00:42<00:10,  6.30it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  81%|████████▉  | 271/334 [00:42<00:09,  6.32it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  81%|████████▉  | 272/334 [00:42<00:09,  6.33it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  82%|████████▉  | 273/334 [00:43<00:09,  6.35it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  82%|█████████  | 274/334 [00:43<00:09,  6.36it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  82%|█████████  | 275/334 [00:43<00:09,  6.38it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  83%|█████████  | 276/334 [00:43<00:09,  6.39it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  83%|█████████  | 277/334 [00:43<00:08,  6.41it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  83%|█████████▏ | 278/334 [00:43<00:08,  6.42it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  84%|█████████▏ | 279/334 [00:43<00:08,  6.44it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  84%|█████████▏ | 280/334 [00:43<00:08,  6.46it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  84%|█████████▎ | 281/334 [00:43<00:08,  6.47it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  84%|█████████▎ | 282/334 [00:43<00:08,  6.49it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  85%|█████████▎ | 283/334 [00:43<00:07,  6.50it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  85%|█████████▎ | 284/334 [00:43<00:07,  6.52it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  85%|█████████▍ | 285/334 [00:43<00:07,  6.53it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  86%|█████████▍ | 286/334 [00:43<00:07,  6.55it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  86%|█████████▍ | 287/334 [00:43<00:07,  6.56it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  86%|█████████▍ | 288/334 [00:43<00:06,  6.58it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  87%|█████████▌ | 289/334 [00:43<00:06,  6.59it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  87%|█████████▌ | 290/334 [00:43<00:06,  6.61it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  87%|█████████▌ | 291/334 [00:43<00:06,  6.62it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  87%|█████████▌ | 292/334 [00:43<00:06,  6.64it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  88%|█████████▋ | 293/334 [00:44<00:06,  6.65it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  88%|█████████▋ | 294/334 [00:44<00:05,  6.67it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  88%|█████████▋ | 295/334 [00:44<00:05,  6.68it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  89%|█████████▋ | 296/334 [00:44<00:05,  6.70it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  89%|█████████▊ | 297/334 [00:44<00:05,  6.71it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  89%|█████████▊ | 298/334 [00:44<00:05,  6.73it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  90%|█████████▊ | 299/334 [00:44<00:05,  6.74it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  90%|█████████▉ | 300/334 [00:44<00:05,  6.76it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  90%|█████████▉ | 301/334 [00:44<00:04,  6.77it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  90%|█████████▉ | 302/334 [00:44<00:04,  6.79it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  91%|█████████▉ | 303/334 [00:44<00:04,  6.80it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  91%|██████████ | 304/334 [00:44<00:04,  6.82it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  91%|██████████ | 305/334 [00:44<00:04,  6.83it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  92%|██████████ | 306/334 [00:44<00:04,  6.85it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  92%|██████████ | 307/334 [00:44<00:03,  6.86it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  92%|██████████▏| 308/334 [00:44<00:03,  6.88it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  93%|██████████▏| 309/334 [00:44<00:03,  6.89it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  93%|██████████▏| 310/334 [00:44<00:03,  6.90it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  93%|██████████▏| 311/334 [00:45<00:03,  6.91it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  93%|██████████▎| 312/334 [00:45<00:03,  6.90it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  94%|██████████▎| 313/334 [00:45<00:03,  6.91it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  94%|██████████▎| 314/334 [00:45<00:02,  6.89it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  94%|██████████▎| 315/334 [00:45<00:02,  6.90it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  95%|██████████▍| 316/334 [00:45<00:02,  6.92it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  95%|██████████▍| 317/334 [00:45<00:02,  6.93it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  95%|██████████▍| 318/334 [00:45<00:02,  6.95it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  96%|██████████▌| 319/334 [00:45<00:02,  6.96it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  96%|██████████▌| 320/334 [00:45<00:02,  6.97it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  96%|██████████▌| 321/334 [00:45<00:01,  6.99it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  96%|██████████▌| 322/334 [00:45<00:01,  7.00it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  97%|██████████▋| 323/334 [00:46<00:01,  7.02it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  97%|██████████▋| 324/334 [00:46<00:01,  7.03it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  97%|██████████▋| 325/334 [00:46<00:01,  7.04it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  98%|██████████▋| 326/334 [00:46<00:01,  7.06it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  98%|██████████▊| 327/334 [00:46<00:00,  7.07it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  98%|██████████▊| 328/334 [00:46<00:00,  7.08it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  99%|██████████▊| 329/334 [00:46<00:00,  7.10it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  99%|██████████▊| 330/334 [00:46<00:00,  7.11it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  99%|██████████▉| 331/334 [00:46<00:00,  7.12it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9:  99%|██████████▉| 332/334 [00:46<00:00,  7.12it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9: 100%|██████████▉| 333/334 [00:46<00:00,  7.13it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9: 100%|███████████| 334/334 [00:46<00:00,  7.15it/s, loss=0.414, v_num=0]\u001b[A\n",
      "Epoch 9: 100%|███████████| 334/334 [00:46<00:00,  7.15it/s, loss=0.414, v_num=0]Epoch 9, global step 2470: 'valid_mac_acc' was not in top 1\n",
      "Epoch 10:  74%|███████▍  | 247/334 [00:41<00:14,  5.97it/s, loss=0.369, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 10:  74%|███████▍  | 248/334 [00:41<00:14,  5.93it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  75%|███████▍  | 249/334 [00:41<00:14,  5.95it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  75%|███████▍  | 250/334 [00:41<00:14,  5.96it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  75%|███████▌  | 251/334 [00:41<00:13,  5.98it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  75%|███████▌  | 252/334 [00:42<00:13,  5.99it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  76%|███████▌  | 253/334 [00:42<00:13,  6.01it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  76%|███████▌  | 254/334 [00:42<00:13,  6.03it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  76%|███████▋  | 255/334 [00:42<00:13,  6.04it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  77%|███████▋  | 256/334 [00:42<00:12,  6.06it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  77%|███████▋  | 257/334 [00:42<00:12,  6.07it/s, loss=0.369, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10:  77%|███████▋  | 258/334 [00:42<00:12,  6.09it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  78%|███████▊  | 259/334 [00:42<00:12,  6.11it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  78%|███████▊  | 260/334 [00:42<00:12,  6.12it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  78%|███████▊  | 261/334 [00:42<00:11,  6.14it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  78%|███████▊  | 262/334 [00:42<00:11,  6.15it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  79%|███████▊  | 263/334 [00:42<00:11,  6.17it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  79%|███████▉  | 264/334 [00:42<00:11,  6.19it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  79%|███████▉  | 265/334 [00:42<00:11,  6.20it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  80%|███████▉  | 266/334 [00:42<00:10,  6.22it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  80%|███████▉  | 267/334 [00:42<00:10,  6.24it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  80%|████████  | 268/334 [00:42<00:10,  6.25it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  81%|████████  | 269/334 [00:42<00:10,  6.27it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  81%|████████  | 270/334 [00:42<00:10,  6.28it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  81%|████████  | 271/334 [00:43<00:10,  6.30it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  81%|████████▏ | 272/334 [00:43<00:09,  6.31it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  82%|████████▏ | 273/334 [00:43<00:09,  6.33it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  82%|████████▏ | 274/334 [00:43<00:09,  6.34it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  82%|████████▏ | 275/334 [00:43<00:09,  6.36it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  83%|████████▎ | 276/334 [00:43<00:09,  6.37it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  83%|████████▎ | 277/334 [00:43<00:08,  6.39it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  83%|████████▎ | 278/334 [00:43<00:08,  6.41it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  84%|████████▎ | 279/334 [00:43<00:08,  6.42it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  84%|████████▍ | 280/334 [00:43<00:08,  6.44it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  84%|████████▍ | 281/334 [00:43<00:08,  6.45it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  84%|████████▍ | 282/334 [00:43<00:08,  6.47it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  85%|████████▍ | 283/334 [00:43<00:07,  6.48it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  85%|████████▌ | 284/334 [00:43<00:07,  6.50it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  85%|████████▌ | 285/334 [00:43<00:07,  6.51it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  86%|████████▌ | 286/334 [00:43<00:07,  6.53it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  86%|████████▌ | 287/334 [00:43<00:07,  6.54it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  86%|████████▌ | 288/334 [00:43<00:07,  6.56it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  87%|████████▋ | 289/334 [00:43<00:06,  6.57it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  87%|████████▋ | 290/334 [00:44<00:06,  6.58it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  87%|████████▋ | 291/334 [00:44<00:06,  6.60it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  87%|████████▋ | 292/334 [00:44<00:06,  6.62it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  88%|████████▊ | 293/334 [00:44<00:06,  6.63it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  88%|████████▊ | 294/334 [00:44<00:06,  6.64it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  88%|████████▊ | 295/334 [00:44<00:05,  6.66it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  89%|████████▊ | 296/334 [00:44<00:05,  6.67it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  89%|████████▉ | 297/334 [00:44<00:05,  6.69it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  89%|████████▉ | 298/334 [00:44<00:05,  6.70it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  90%|████████▉ | 299/334 [00:44<00:05,  6.72it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  90%|████████▉ | 300/334 [00:44<00:05,  6.73it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  90%|█████████ | 301/334 [00:44<00:04,  6.75it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  90%|█████████ | 302/334 [00:44<00:04,  6.76it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  91%|█████████ | 303/334 [00:44<00:04,  6.78it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  91%|█████████ | 304/334 [00:44<00:04,  6.79it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  91%|█████████▏| 305/334 [00:44<00:04,  6.81it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  92%|█████████▏| 306/334 [00:44<00:04,  6.82it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  92%|█████████▏| 307/334 [00:44<00:03,  6.84it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  92%|█████████▏| 308/334 [00:44<00:03,  6.85it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  93%|█████████▎| 309/334 [00:45<00:03,  6.86it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  93%|█████████▎| 310/334 [00:45<00:03,  6.88it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  93%|█████████▎| 311/334 [00:45<00:03,  6.88it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  93%|█████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  94%|█████████▎| 313/334 [00:45<00:03,  6.89it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  94%|█████████▍| 314/334 [00:45<00:02,  6.87it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  94%|█████████▍| 315/334 [00:45<00:02,  6.88it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  95%|█████████▍| 316/334 [00:45<00:02,  6.89it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  95%|█████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  95%|█████████▌| 318/334 [00:45<00:02,  6.92it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  96%|█████████▌| 319/334 [00:45<00:02,  6.94it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  96%|█████████▌| 320/334 [00:46<00:02,  6.95it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  96%|█████████▌| 321/334 [00:46<00:01,  6.96it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  96%|█████████▋| 322/334 [00:46<00:01,  6.98it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  97%|█████████▋| 323/334 [00:46<00:01,  6.99it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  97%|█████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  97%|█████████▋| 325/334 [00:46<00:01,  7.02it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  98%|█████████▊| 326/334 [00:46<00:01,  7.03it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  98%|█████████▊| 327/334 [00:46<00:00,  7.05it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  98%|█████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  99%|█████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  99%|█████████▉| 330/334 [00:46<00:00,  7.09it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  99%|█████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10:  99%|█████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10: 100%|█████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 10: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.369, v_num=0]Epoch 10, global step 2717: 'valid_mac_acc' reached 91.13596 (best 91.13596), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=10-valid_mac_acc=91.14.ckpt' as top 1\n",
      "Epoch 11:  74%|███████▍  | 247/334 [00:41<00:14,  5.96it/s, loss=0.356, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 11:  74%|███████▍  | 248/334 [00:41<00:14,  5.92it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  75%|███████▍  | 249/334 [00:41<00:14,  5.94it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  75%|███████▍  | 250/334 [00:42<00:14,  5.95it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  75%|███████▌  | 251/334 [00:42<00:13,  5.97it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  75%|███████▌  | 252/334 [00:42<00:13,  5.98it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  76%|███████▌  | 253/334 [00:42<00:13,  6.00it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  76%|███████▌  | 254/334 [00:42<00:13,  6.02it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  76%|███████▋  | 255/334 [00:42<00:13,  6.03it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  77%|███████▋  | 256/334 [00:42<00:12,  6.05it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  77%|███████▋  | 257/334 [00:42<00:12,  6.06it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  77%|███████▋  | 258/334 [00:42<00:12,  6.08it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  78%|███████▊  | 259/334 [00:42<00:12,  6.10it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  78%|███████▊  | 260/334 [00:42<00:12,  6.11it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  78%|███████▊  | 261/334 [00:42<00:11,  6.13it/s, loss=0.356, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11:  78%|███████▊  | 262/334 [00:42<00:11,  6.15it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  79%|███████▊  | 263/334 [00:42<00:11,  6.16it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  79%|███████▉  | 264/334 [00:42<00:11,  6.18it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  79%|███████▉  | 265/334 [00:42<00:11,  6.19it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  80%|███████▉  | 266/334 [00:42<00:10,  6.21it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  80%|███████▉  | 267/334 [00:42<00:10,  6.23it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  80%|████████  | 268/334 [00:42<00:10,  6.24it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  81%|████████  | 269/334 [00:42<00:10,  6.26it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  81%|████████  | 270/334 [00:43<00:10,  6.27it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  81%|████████  | 271/334 [00:43<00:10,  6.29it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  81%|████████▏ | 272/334 [00:43<00:09,  6.30it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  82%|████████▏ | 273/334 [00:43<00:09,  6.32it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  82%|████████▏ | 274/334 [00:43<00:09,  6.34it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  82%|████████▏ | 275/334 [00:43<00:09,  6.35it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  83%|████████▎ | 276/334 [00:43<00:09,  6.37it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  83%|████████▎ | 277/334 [00:43<00:08,  6.38it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  83%|████████▎ | 278/334 [00:43<00:08,  6.40it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  84%|████████▎ | 279/334 [00:43<00:08,  6.41it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  84%|████████▍ | 280/334 [00:43<00:08,  6.43it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  84%|████████▍ | 281/334 [00:43<00:08,  6.44it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  84%|████████▍ | 282/334 [00:43<00:08,  6.46it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  85%|████████▍ | 283/334 [00:43<00:07,  6.47it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  85%|████████▌ | 284/334 [00:43<00:07,  6.49it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  85%|████████▌ | 285/334 [00:43<00:07,  6.50it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  86%|████████▌ | 286/334 [00:43<00:07,  6.52it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  86%|████████▌ | 287/334 [00:43<00:07,  6.53it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  86%|████████▌ | 288/334 [00:43<00:07,  6.55it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  87%|████████▋ | 289/334 [00:44<00:06,  6.56it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  87%|████████▋ | 290/334 [00:44<00:06,  6.58it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  87%|████████▋ | 291/334 [00:44<00:06,  6.59it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  87%|████████▋ | 292/334 [00:44<00:06,  6.61it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  88%|████████▊ | 293/334 [00:44<00:06,  6.62it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  88%|████████▊ | 294/334 [00:44<00:06,  6.64it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  88%|████████▊ | 295/334 [00:44<00:05,  6.65it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  89%|████████▊ | 296/334 [00:44<00:05,  6.67it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  89%|████████▉ | 297/334 [00:44<00:05,  6.68it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  89%|████████▉ | 298/334 [00:44<00:05,  6.70it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  90%|████████▉ | 299/334 [00:44<00:05,  6.71it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  90%|████████▉ | 300/334 [00:44<00:05,  6.73it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  90%|█████████ | 301/334 [00:44<00:04,  6.74it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  90%|█████████ | 302/334 [00:44<00:04,  6.76it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  91%|█████████ | 303/334 [00:44<00:04,  6.77it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  91%|█████████ | 304/334 [00:44<00:04,  6.79it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  91%|█████████▏| 305/334 [00:44<00:04,  6.80it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  92%|█████████▏| 306/334 [00:44<00:04,  6.81it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  92%|█████████▏| 307/334 [00:44<00:03,  6.83it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  92%|█████████▏| 308/334 [00:45<00:03,  6.84it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  93%|█████████▎| 309/334 [00:45<00:03,  6.86it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  93%|█████████▎| 310/334 [00:45<00:03,  6.87it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  93%|█████████▎| 311/334 [00:45<00:03,  6.88it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  93%|█████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  94%|█████████▎| 313/334 [00:45<00:03,  6.88it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  94%|█████████▍| 314/334 [00:45<00:02,  6.86it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  94%|█████████▍| 315/334 [00:45<00:02,  6.87it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  95%|█████████▍| 316/334 [00:45<00:02,  6.89it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  95%|█████████▍| 317/334 [00:45<00:02,  6.90it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  95%|█████████▌| 318/334 [00:45<00:02,  6.91it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  96%|█████████▌| 319/334 [00:46<00:02,  6.93it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  96%|█████████▌| 320/334 [00:46<00:02,  6.94it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  96%|█████████▌| 321/334 [00:46<00:01,  6.96it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  96%|█████████▋| 322/334 [00:46<00:01,  6.97it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  97%|█████████▋| 323/334 [00:46<00:01,  6.98it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  97%|█████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  97%|█████████▋| 325/334 [00:46<00:01,  7.01it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  98%|█████████▊| 326/334 [00:46<00:01,  7.02it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  98%|█████████▊| 327/334 [00:46<00:00,  7.04it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  98%|█████████▊| 328/334 [00:46<00:00,  7.05it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  99%|█████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  99%|█████████▉| 330/334 [00:46<00:00,  7.08it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  99%|█████████▉| 331/334 [00:46<00:00,  7.09it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11:  99%|█████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11: 100%|█████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11: 100%|██████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.356, v_num=0]\u001b[A\n",
      "Epoch 11: 100%|██████████| 334/334 [00:46<00:00,  7.11it/s, loss=0.356, v_num=0]Epoch 11, global step 2964: 'valid_mac_acc' reached 91.27733 (best 91.27733), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=11-valid_mac_acc=91.28.ckpt' as top 1\n",
      "Epoch 12:  74%|███████▍  | 247/334 [00:41<00:14,  5.97it/s, loss=0.369, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 12:  74%|███████▍  | 248/334 [00:41<00:14,  5.93it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  75%|███████▍  | 249/334 [00:41<00:14,  5.94it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  75%|███████▍  | 250/334 [00:41<00:14,  5.96it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  75%|███████▌  | 251/334 [00:41<00:13,  5.98it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  75%|███████▌  | 252/334 [00:42<00:13,  5.99it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  76%|███████▌  | 253/334 [00:42<00:13,  6.01it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  76%|███████▌  | 254/334 [00:42<00:13,  6.03it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  76%|███████▋  | 255/334 [00:42<00:13,  6.04it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  77%|███████▋  | 256/334 [00:42<00:12,  6.06it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  77%|███████▋  | 257/334 [00:42<00:12,  6.08it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  77%|███████▋  | 258/334 [00:42<00:12,  6.09it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  78%|███████▊  | 259/334 [00:42<00:12,  6.11it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  78%|███████▊  | 260/334 [00:42<00:12,  6.12it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  78%|███████▊  | 261/334 [00:42<00:11,  6.14it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  78%|███████▊  | 262/334 [00:42<00:11,  6.16it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  79%|███████▊  | 263/334 [00:42<00:11,  6.17it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  79%|███████▉  | 264/334 [00:42<00:11,  6.19it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  79%|███████▉  | 265/334 [00:42<00:11,  6.21it/s, loss=0.369, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12:  80%|███████▉  | 266/334 [00:42<00:10,  6.22it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  80%|███████▉  | 267/334 [00:42<00:10,  6.24it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  80%|████████  | 268/334 [00:42<00:10,  6.25it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  81%|████████  | 269/334 [00:42<00:10,  6.27it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  81%|████████  | 270/334 [00:42<00:10,  6.29it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  81%|████████  | 271/334 [00:43<00:09,  6.30it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  81%|████████▏ | 272/334 [00:43<00:09,  6.32it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  82%|████████▏ | 273/334 [00:43<00:09,  6.33it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  82%|████████▏ | 274/334 [00:43<00:09,  6.35it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  82%|████████▏ | 275/334 [00:43<00:09,  6.36it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  83%|████████▎ | 276/334 [00:43<00:09,  6.38it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  83%|████████▎ | 277/334 [00:43<00:08,  6.39it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  83%|████████▎ | 278/334 [00:43<00:08,  6.41it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  84%|████████▎ | 279/334 [00:43<00:08,  6.42it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  84%|████████▍ | 280/334 [00:43<00:08,  6.44it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  84%|████████▍ | 281/334 [00:43<00:08,  6.45it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  84%|████████▍ | 282/334 [00:43<00:08,  6.47it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  85%|████████▍ | 283/334 [00:43<00:07,  6.49it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  85%|████████▌ | 284/334 [00:43<00:07,  6.50it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  85%|████████▌ | 285/334 [00:43<00:07,  6.52it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  86%|████████▌ | 286/334 [00:43<00:07,  6.53it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  86%|████████▌ | 287/334 [00:43<00:07,  6.55it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  86%|████████▌ | 288/334 [00:43<00:07,  6.56it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  87%|████████▋ | 289/334 [00:43<00:06,  6.58it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  87%|████████▋ | 290/334 [00:44<00:06,  6.59it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  87%|████████▋ | 291/334 [00:44<00:06,  6.61it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  87%|████████▋ | 292/334 [00:44<00:06,  6.62it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  88%|████████▊ | 293/334 [00:44<00:06,  6.63it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  88%|████████▊ | 294/334 [00:44<00:06,  6.65it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  88%|████████▊ | 295/334 [00:44<00:05,  6.66it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  89%|████████▊ | 296/334 [00:44<00:05,  6.68it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  89%|████████▉ | 297/334 [00:44<00:05,  6.69it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  89%|████████▉ | 298/334 [00:44<00:05,  6.71it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  90%|████████▉ | 299/334 [00:44<00:05,  6.72it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  90%|████████▉ | 300/334 [00:44<00:05,  6.74it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  90%|█████████ | 301/334 [00:44<00:04,  6.75it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  90%|█████████ | 302/334 [00:44<00:04,  6.77it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  91%|█████████ | 303/334 [00:44<00:04,  6.78it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  91%|█████████ | 304/334 [00:44<00:04,  6.80it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  91%|█████████▏| 305/334 [00:44<00:04,  6.81it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  92%|█████████▏| 306/334 [00:44<00:04,  6.83it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  92%|█████████▏| 307/334 [00:44<00:03,  6.84it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  92%|█████████▏| 308/334 [00:44<00:03,  6.85it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  93%|█████████▎| 309/334 [00:44<00:03,  6.87it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  93%|█████████▎| 310/334 [00:45<00:03,  6.88it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  93%|█████████▎| 311/334 [00:45<00:03,  6.89it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  93%|█████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  94%|█████████▎| 313/334 [00:45<00:03,  6.89it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  94%|█████████▍| 314/334 [00:45<00:02,  6.87it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  94%|█████████▍| 315/334 [00:45<00:02,  6.88it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  95%|█████████▍| 316/334 [00:45<00:02,  6.90it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  95%|█████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  95%|█████████▌| 318/334 [00:45<00:02,  6.93it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  96%|█████████▌| 319/334 [00:45<00:02,  6.94it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  96%|█████████▌| 320/334 [00:46<00:02,  6.95it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  96%|█████████▌| 321/334 [00:46<00:01,  6.97it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  96%|█████████▋| 322/334 [00:46<00:01,  6.98it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  97%|█████████▋| 323/334 [00:46<00:01,  7.00it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  97%|█████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  97%|█████████▋| 325/334 [00:46<00:01,  7.02it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  98%|█████████▊| 326/334 [00:46<00:01,  7.03it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  98%|█████████▊| 327/334 [00:46<00:00,  7.05it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  98%|█████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  99%|█████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  99%|█████████▉| 330/334 [00:46<00:00,  7.09it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  99%|█████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12:  99%|█████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12: 100%|█████████▉| 333/334 [00:46<00:00,  7.10it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.369, v_num=0]\u001b[A\n",
      "Epoch 12: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.369, v_num=0]Epoch 12, global step 3211: 'valid_mac_acc' was not in top 1\n",
      "Epoch 13:  74%|███████▍  | 247/334 [00:41<00:14,  5.95it/s, loss=0.376, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 13:  74%|███████▍  | 248/334 [00:41<00:14,  5.91it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  75%|███████▍  | 249/334 [00:42<00:14,  5.92it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  75%|███████▍  | 250/334 [00:42<00:14,  5.94it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  75%|███████▌  | 251/334 [00:42<00:13,  5.96it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  75%|███████▌  | 252/334 [00:42<00:13,  5.97it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  76%|███████▌  | 253/334 [00:42<00:13,  5.99it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  76%|███████▌  | 254/334 [00:42<00:13,  6.00it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  76%|███████▋  | 255/334 [00:42<00:13,  6.02it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  77%|███████▋  | 256/334 [00:42<00:12,  6.04it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  77%|███████▋  | 257/334 [00:42<00:12,  6.05it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  77%|███████▋  | 258/334 [00:42<00:12,  6.07it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  78%|███████▊  | 259/334 [00:42<00:12,  6.09it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  78%|███████▊  | 260/334 [00:42<00:12,  6.10it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  78%|███████▊  | 261/334 [00:42<00:11,  6.12it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  78%|███████▊  | 262/334 [00:42<00:11,  6.14it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  79%|███████▊  | 263/334 [00:42<00:11,  6.15it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  79%|███████▉  | 264/334 [00:42<00:11,  6.17it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  79%|███████▉  | 265/334 [00:42<00:11,  6.18it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  80%|███████▉  | 266/334 [00:42<00:10,  6.20it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  80%|███████▉  | 267/334 [00:42<00:10,  6.22it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  80%|████████  | 268/334 [00:43<00:10,  6.23it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  81%|████████  | 269/334 [00:43<00:10,  6.25it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  81%|████████  | 270/334 [00:43<00:10,  6.26it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  81%|████████  | 271/334 [00:43<00:10,  6.28it/s, loss=0.376, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13:  81%|████████▏ | 272/334 [00:43<00:09,  6.29it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  82%|████████▏ | 273/334 [00:43<00:09,  6.31it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  82%|████████▏ | 274/334 [00:43<00:09,  6.33it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  82%|████████▏ | 275/334 [00:43<00:09,  6.34it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  83%|████████▎ | 276/334 [00:43<00:09,  6.36it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  83%|████████▎ | 277/334 [00:43<00:08,  6.37it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  83%|████████▎ | 278/334 [00:43<00:08,  6.39it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  84%|████████▎ | 279/334 [00:43<00:08,  6.40it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  84%|████████▍ | 280/334 [00:43<00:08,  6.42it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  84%|████████▍ | 281/334 [00:43<00:08,  6.43it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  84%|████████▍ | 282/334 [00:43<00:08,  6.45it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  85%|████████▍ | 283/334 [00:43<00:07,  6.47it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  85%|████████▌ | 284/334 [00:43<00:07,  6.48it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  85%|████████▌ | 285/334 [00:43<00:07,  6.50it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  86%|████████▌ | 286/334 [00:43<00:07,  6.51it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  86%|████████▌ | 287/334 [00:43<00:07,  6.53it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  86%|████████▌ | 288/334 [00:44<00:07,  6.54it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  87%|████████▋ | 289/334 [00:44<00:06,  6.56it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  87%|████████▋ | 290/334 [00:44<00:06,  6.57it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  87%|████████▋ | 291/334 [00:44<00:06,  6.59it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  87%|████████▋ | 292/334 [00:44<00:06,  6.60it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  88%|████████▊ | 293/334 [00:44<00:06,  6.62it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  88%|████████▊ | 294/334 [00:44<00:06,  6.63it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  88%|████████▊ | 295/334 [00:44<00:05,  6.65it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  89%|████████▊ | 296/334 [00:44<00:05,  6.66it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  89%|████████▉ | 297/334 [00:44<00:05,  6.68it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  89%|████████▉ | 298/334 [00:44<00:05,  6.69it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  90%|████████▉ | 299/334 [00:44<00:05,  6.71it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  90%|████████▉ | 300/334 [00:44<00:05,  6.72it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  90%|█████████ | 301/334 [00:44<00:04,  6.74it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  90%|█████████ | 302/334 [00:44<00:04,  6.75it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  91%|█████████ | 303/334 [00:44<00:04,  6.77it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  91%|█████████ | 304/334 [00:44<00:04,  6.78it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  91%|█████████▏| 305/334 [00:44<00:04,  6.79it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  92%|█████████▏| 306/334 [00:44<00:04,  6.81it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  92%|█████████▏| 307/334 [00:44<00:03,  6.82it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  92%|█████████▏| 308/334 [00:45<00:03,  6.84it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  93%|█████████▎| 309/334 [00:45<00:03,  6.85it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  93%|█████████▎| 310/334 [00:45<00:03,  6.87it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  93%|█████████▎| 311/334 [00:45<00:03,  6.87it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  93%|█████████▎| 312/334 [00:45<00:03,  6.86it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  94%|█████████▎| 313/334 [00:45<00:03,  6.88it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  94%|█████████▍| 314/334 [00:45<00:02,  6.85it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  94%|█████████▍| 315/334 [00:45<00:02,  6.87it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  95%|█████████▍| 316/334 [00:45<00:02,  6.88it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  95%|█████████▍| 317/334 [00:45<00:02,  6.89it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  95%|█████████▌| 318/334 [00:46<00:02,  6.91it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  96%|█████████▌| 319/334 [00:46<00:02,  6.92it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  96%|█████████▌| 320/334 [00:46<00:02,  6.94it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  96%|█████████▌| 321/334 [00:46<00:01,  6.95it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  96%|█████████▋| 322/334 [00:46<00:01,  6.96it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  97%|█████████▋| 323/334 [00:46<00:01,  6.98it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  97%|█████████▋| 324/334 [00:46<00:01,  6.99it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  97%|█████████▋| 325/334 [00:46<00:01,  7.01it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  98%|█████████▊| 326/334 [00:46<00:01,  7.02it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  98%|█████████▊| 327/334 [00:46<00:00,  7.03it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  98%|█████████▊| 328/334 [00:46<00:00,  7.05it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  99%|█████████▊| 329/334 [00:46<00:00,  7.06it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  99%|█████████▉| 330/334 [00:46<00:00,  7.07it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  99%|█████████▉| 331/334 [00:46<00:00,  7.09it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13:  99%|█████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13: 100%|█████████▉| 333/334 [00:46<00:00,  7.11it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.376, v_num=0]\u001b[A\n",
      "Epoch 13: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.376, v_num=0]Epoch 13, global step 3458: 'valid_mac_acc' was not in top 1\n",
      "Epoch 14:  74%|███████▍  | 247/334 [00:41<00:14,  5.96it/s, loss=0.367, v_num=0]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                        | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                           | 0/87 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 14:  74%|███████▍  | 248/334 [00:41<00:14,  5.92it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  75%|███████▍  | 249/334 [00:41<00:14,  5.94it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  75%|███████▍  | 250/334 [00:41<00:14,  5.95it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  75%|███████▌  | 251/334 [00:42<00:13,  5.97it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  75%|███████▌  | 252/334 [00:42<00:13,  5.99it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  76%|███████▌  | 253/334 [00:42<00:13,  6.00it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  76%|███████▌  | 254/334 [00:42<00:13,  6.02it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  76%|███████▋  | 255/334 [00:42<00:13,  6.04it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  77%|███████▋  | 256/334 [00:42<00:12,  6.05it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  77%|███████▋  | 257/334 [00:42<00:12,  6.07it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  77%|███████▋  | 258/334 [00:42<00:12,  6.09it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  78%|███████▊  | 259/334 [00:42<00:12,  6.10it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  78%|███████▊  | 260/334 [00:42<00:12,  6.12it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  78%|███████▊  | 261/334 [00:42<00:11,  6.13it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  78%|███████▊  | 262/334 [00:42<00:11,  6.15it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  79%|███████▊  | 263/334 [00:42<00:11,  6.17it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  79%|███████▉  | 264/334 [00:42<00:11,  6.18it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  79%|███████▉  | 265/334 [00:42<00:11,  6.20it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  80%|███████▉  | 266/334 [00:42<00:10,  6.21it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  80%|███████▉  | 267/334 [00:42<00:10,  6.23it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  80%|████████  | 268/334 [00:42<00:10,  6.25it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  81%|████████  | 269/334 [00:42<00:10,  6.26it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  81%|████████  | 270/334 [00:43<00:10,  6.28it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  81%|████████  | 271/334 [00:43<00:10,  6.29it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  81%|████████▏ | 272/334 [00:43<00:09,  6.31it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  82%|████████▏ | 273/334 [00:43<00:09,  6.32it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  82%|████████▏ | 274/334 [00:43<00:09,  6.34it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  82%|████████▏ | 275/334 [00:43<00:09,  6.35it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  83%|████████▎ | 276/334 [00:43<00:09,  6.37it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  83%|████████▎ | 277/334 [00:43<00:08,  6.39it/s, loss=0.367, v_num=0]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14:  83%|████████▎ | 278/334 [00:43<00:08,  6.40it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  84%|████████▎ | 279/334 [00:43<00:08,  6.42it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  84%|████████▍ | 280/334 [00:43<00:08,  6.43it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  84%|████████▍ | 281/334 [00:43<00:08,  6.45it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  84%|████████▍ | 282/334 [00:43<00:08,  6.46it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  85%|████████▍ | 283/334 [00:43<00:07,  6.48it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  85%|████████▌ | 284/334 [00:43<00:07,  6.49it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  85%|████████▌ | 285/334 [00:43<00:07,  6.51it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  86%|████████▌ | 286/334 [00:43<00:07,  6.52it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  86%|████████▌ | 287/334 [00:43<00:07,  6.54it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  86%|████████▌ | 288/334 [00:43<00:07,  6.55it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  87%|████████▋ | 289/334 [00:43<00:06,  6.57it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  87%|████████▋ | 290/334 [00:44<00:06,  6.58it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  87%|████████▋ | 291/334 [00:44<00:06,  6.60it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  87%|████████▋ | 292/334 [00:44<00:06,  6.61it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  88%|████████▊ | 293/334 [00:44<00:06,  6.63it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  88%|████████▊ | 294/334 [00:44<00:06,  6.64it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  88%|████████▊ | 295/334 [00:44<00:05,  6.66it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  89%|████████▊ | 296/334 [00:44<00:05,  6.67it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  89%|████████▉ | 297/334 [00:44<00:05,  6.69it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  89%|████████▉ | 298/334 [00:44<00:05,  6.70it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  90%|████████▉ | 299/334 [00:44<00:05,  6.72it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  90%|████████▉ | 300/334 [00:44<00:05,  6.73it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  90%|█████████ | 301/334 [00:44<00:04,  6.75it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  90%|█████████ | 302/334 [00:44<00:04,  6.76it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  91%|█████████ | 303/334 [00:44<00:04,  6.78it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  91%|█████████ | 304/334 [00:44<00:04,  6.79it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  91%|█████████▏| 305/334 [00:44<00:04,  6.81it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  92%|█████████▏| 306/334 [00:44<00:04,  6.82it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  92%|█████████▏| 307/334 [00:44<00:03,  6.83it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  92%|█████████▏| 308/334 [00:44<00:03,  6.85it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  93%|█████████▎| 309/334 [00:45<00:03,  6.86it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  93%|█████████▎| 310/334 [00:45<00:03,  6.88it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  93%|█████████▎| 311/334 [00:45<00:03,  6.88it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  93%|█████████▎| 312/334 [00:45<00:03,  6.87it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  94%|█████████▎| 313/334 [00:45<00:03,  6.88it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  94%|█████████▍| 314/334 [00:45<00:02,  6.86it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  94%|█████████▍| 315/334 [00:45<00:02,  6.88it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  95%|█████████▍| 316/334 [00:45<00:02,  6.89it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  95%|█████████▍| 317/334 [00:45<00:02,  6.91it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  95%|█████████▌| 318/334 [00:45<00:02,  6.92it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  96%|█████████▌| 319/334 [00:46<00:02,  6.93it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  96%|█████████▌| 320/334 [00:46<00:02,  6.95it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  96%|█████████▌| 321/334 [00:46<00:01,  6.96it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  96%|█████████▋| 322/334 [00:46<00:01,  6.98it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  97%|█████████▋| 323/334 [00:46<00:01,  6.99it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  97%|█████████▋| 324/334 [00:46<00:01,  7.00it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  97%|█████████▋| 325/334 [00:46<00:01,  7.02it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  98%|█████████▊| 326/334 [00:46<00:01,  7.03it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  98%|█████████▊| 327/334 [00:46<00:00,  7.04it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  98%|█████████▊| 328/334 [00:46<00:00,  7.06it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  99%|█████████▊| 329/334 [00:46<00:00,  7.07it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  99%|█████████▉| 330/334 [00:46<00:00,  7.08it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  99%|█████████▉| 331/334 [00:46<00:00,  7.10it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14:  99%|█████████▉| 332/334 [00:46<00:00,  7.09it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14: 100%|█████████▉| 333/334 [00:46<00:00,  7.11it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.367, v_num=0]\u001b[A\n",
      "Epoch 14: 100%|██████████| 334/334 [00:46<00:00,  7.12it/s, loss=0.367, v_num=0]Epoch 14, global step 3705: 'valid_mac_acc' reached 91.38474 (best 91.38474), saving model to './weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=14-valid_mac_acc=91.38.ckpt' as top 1\n",
      "`Trainer.fit` stopped: `max_epochs=15` reached.\n",
      "Epoch 14: 100%|██████████| 334/334 [00:47<00:00,  7.03it/s, loss=0.367, v_num=0]\n",
      "(PT_Finetuning) "
     ]
    },
    {
     "ename": "",
     "evalue": "1",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "export SLURM_NTASKS_PER_NODE=24\n",
    "\n",
    "python main.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "782b2672",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(PT_Finetuning) (PT_Finetuning) [rank: 0] Global seed set to 0\n",
      "Fusing layers... \n",
      "Fusing layers... \n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "100%|█████████████████████████████████████████| 656/656 [58:18<00:00,  5.33s/it]\n",
      "Fusing layers... \n",
      "Fusing layers... \n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "100%|█████████████████████████████████████████| 656/656 [58:19<00:00,  5.34s/it]\n",
      "Fusing layers... \n",
      "Fusing layers... \n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "Model summary: 733 layers, 140054656 parameters, 0 gradients, 208.8 GFLOPs\n",
      "100%|█████████████████████████████████████████| 656/656 [57:40<00:00,  5.28s/it]\n",
      "Number of images loaded:  7898\n",
      "Number of images loaded:  2769\n",
      "Number of images loaded:  2758\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "Restoring states from the checkpoint path at ./weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=14-valid_mac_acc=91.38.ckpt\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "Loaded model weights from checkpoint at ./weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=14-valid_mac_acc=91.38.ckpt\n",
      "Testing DataLoader 0: 100%|█████████████████████| 11/11 [00:06<00:00,  1.62it/s]Test output saved to ./weights/Crop/Plain/Crop_res50_plain_082723-0-epoch=14-valid_mac_acc=91.38eval.npz.\n",
      "Testing DataLoader 0: 100%|█████████████████████| 11/11 [00:06<00:00,  1.60it/s]\n",
      "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
      "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
      "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
      "│\u001b[36m \u001b[0m\u001b[36m      valid_mac_acc      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m    88.93272125287388    \u001b[0m\u001b[35m \u001b[0m│\n",
      "│\u001b[36m \u001b[0m\u001b[36m      valid_mic_acc      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m    92.16823785351704    \u001b[0m\u001b[35m \u001b[0m│\n",
      "└───────────────────────────┴───────────────────────────┘\n",
      "(PT_Finetuning) "
     ]
    },
    {
     "ename": "",
     "evalue": "1",
     "output_type": "error",
     "traceback": []
    }
   ],
   "source": [
    "export SLURM_NTASKS_PER_NODE=24\n",
    "\n",
    "python evaluate.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e8c4828",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Bash",
   "language": "bash",
   "name": "bash"
  },
  "language_info": {
   "codemirror_mode": "shell",
   "file_extension": ".sh",
   "mimetype": "text/x-sh",
   "name": "bash"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}