learn_pytorch_34_pytorch实战项目之自定义卷积神经网络kaggle猫狗数据集训练与测试.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1 导入需要的模块\n",
    "import torchvision\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "import torch.nn as nn\n",
    "from torch.optim import sgd, adam\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "transform_train = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize((256, 256)),\n",
    "        transforms.RandomCrop((224, 224)),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "\n",
    "    ]\n",
    ")\n",
    "\n",
    "transform_test = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize((256, 256)),\n",
    "        transforms.RandomCrop((224, 224)),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "\n",
    "    ]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3 训练集和测试集的上一级路径 \n",
    "image_path = \"/content/drive/My Drive\" \n",
    "# image_path = \"D:\\\\迅雷下载\\\\dogs-vs-cats-redux-kernels-edition\"  # windows上的路径 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4 训练集和测试集内容以及标签获取类 \n",
    "class CatDogProcessor(data.Dataset):\n",
    "    def __init__(self, mode, directory):\n",
    "        self.mode = mode\n",
    "        self.list_img = []\n",
    "        self.list_label = []\n",
    "        self.data_size = 0\n",
    "\n",
    "        if self.mode == \"train\":\n",
    "            self.transform = transform_train\n",
    "            directory = directory + \"/sdh_train/\"\n",
    "            for file in os.listdir(directory):\n",
    "                self.list_img.append(directory + file)\n",
    "                self.data_size += 1\n",
    "                name = file.split(sep=\".\")\n",
    "                if name[0] == \"cat\":\n",
    "                    self.list_label.append(0)\n",
    "                elif name[0] == \"dog\":\n",
    "                    self.list_label.append(1)\n",
    "\n",
    "        elif self.mode == \"test\":\n",
    "            self.transform = transform_test\n",
    "            directory = directory + \"/sdh_test/\"\n",
    "            for file in os.listdir(directory):\n",
    "                self.list_img.append(directory + file)\n",
    "                self.data_size += 1\n",
    "                name = file.split(sep=\".\")\n",
    "                if name[0] == \"cat\":\n",
    "                    self.list_label.append(0)\n",
    "                elif name[0] == \"dog\":\n",
    "                    self.list_label.append(1)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        if self.mode == \"train\":\n",
    "            img = Image.open(self.list_img[item])\n",
    "            label = self.list_label[item]\n",
    "            return self.transform(img), torch.tensor([label])\n",
    "        elif self.mode == \"test\":\n",
    "            img = Image.open(self.list_img[item])\n",
    "            label = self.list_label[item]\n",
    "            return self.transform(img), torch.tensor([label])\n",
    "        else:\n",
    "            return None\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.data_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    " # 5 定义一个神经网络模型\n",
    "class SelfMadeCNN(nn.Module):\n",
    "    # 5个卷积层＋2个全连接层\n",
    "    def __init__(self, n_hidden_1, n_hidden_2,out_dim):\n",
    "        super(SelfMadeCNN, self).__init__()\n",
    "        self.layer1 = nn.Sequential(\n",
    "            # (224-3+2*1)/1 +1 = 224\n",
    "            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(32),  # 批标准化放在卷积层后面，激活层前面\n",
    "            nn.ReLU(inplace=True),\n",
    "            # 224/2 = 112\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.layer2 = nn.Sequential(\n",
    "            # (112-3+2*0)/1 +1 = 110\n",
    "            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=0),\n",
    "            nn.BatchNorm2d(64),  # 批标准化放在卷积层后面，激活层前面\n",
    "            nn.ReLU(inplace=True),\n",
    "            # (110-2)/2 +1 = 55\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "\n",
    "        self.layer3 = nn.Sequential(\n",
    "            #  55-2+2*0+1=54\n",
    "            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=2, stride=1, padding=0),\n",
    "            nn.BatchNorm2d(64),  # 批标准化放在卷积层后面，激活层前面\n",
    "            nn.ReLU(inplace=True),\n",
    "            # (54-2)/2 +1 = 27\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "\n",
    "        self.layer4 = nn.Sequential(\n",
    "            #  27-2+2*0+1=26\n",
    "            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=2, stride=1, padding=0),\n",
    "            nn.BatchNorm2d(64),  # 批标准化放在卷积层后面，激活层前面\n",
    "            nn.ReLU(inplace=True),\n",
    "            # (26-2)/2 +1 = 13\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.layer5 = nn.Sequential(\n",
    "            #  13-2+2*0+1=12\n",
    "            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=2, stride=1, padding=0),\n",
    "            nn.BatchNorm2d(64),  # 批标准化放在卷积层后面，激活层前面\n",
    "            nn.ReLU(inplace=True),\n",
    "            # (12-2)/2 +1 = 6\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.layer6 = nn.Sequential(\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Linear(6 * 6 * 64, n_hidden_1),\n",
    "            nn.ReLU(inplace=True),\n",
    "\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Linear(n_hidden_1, n_hidden_2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            \n",
    "            nn.Linear(n_hidden_2, out_dim),\n",
    "\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        # 卷积层\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        x = self.layer4(x)\n",
    "        x = self.layer5(x)\n",
    "\n",
    "        # 全连接层\n",
    "        # 进入全连接之前要将矩阵转为向量\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.layer6(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_image_train(acces, train_losses):\n",
    "    plt.plot(np.arange(len(acces)), acces)\n",
    "    plt.title('train acc')\n",
    "    plt.show()\n",
    "\n",
    "    plt.plot(np.arange(len(train_losses)), train_losses)\n",
    "    plt.title('train loss')\n",
    "    plt.show()\n",
    "\n",
    "def plot_image_test(acces, test_losses):\n",
    "    plt.plot(np.arange(len(acces)), acces)\n",
    "    plt.title('test acc')\n",
    "    plt.show()\n",
    "\n",
    "    plt.plot(np.arange(len(test_losses)), test_losses)\n",
    "    plt.title('test loss')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train():\n",
    "    train_image_data_set = CatDogProcessor(\"train\", image_path)\n",
    "    train_data = DataLoader(train_image_data_set, batch_size=16, shuffle=True)\n",
    "    print(\"train_data loaded\", len(train_data))\n",
    "\n",
    "    test_image_data_set = CatDogProcessor(\"test\", image_path)\n",
    "    test_data = DataLoader(test_image_data_set, batch_size=16, shuffle=True)\n",
    "    print(\"train_data loaded\", len(test_data))\n",
    "\n",
    "    model = SelfMadeCNN(100, 100, 2)\n",
    "    model.to(device)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = sgd.SGD(model.parameters(), lr=0.001,momentum=0.9)\n",
    "\n",
    "    train_losses = []\n",
    "    train_acces = []\n",
    "    test_losses = []\n",
    "    test_acces = []\n",
    "\n",
    "    for i in range(300):\n",
    "        train_loss = 0\n",
    "        train_acc = 0\n",
    "        model.train()\n",
    "        for im, label in train_data:\n",
    "            # 前向传播\n",
    "            im = im.cuda()\n",
    "            label = label.cuda()\n",
    "            label = label.squeeze()\n",
    "            out = model(im)\n",
    "            loss = criterion(out, label)\n",
    "\n",
    "            # 反向传播\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # 记录误差\n",
    "            train_loss += loss.item()\n",
    "            # 计算分类的准确率\n",
    "            _, pred = torch.max(out, 1)\n",
    "            num_correct = (pred == label).sum().item()\n",
    "            acc = num_correct / im.shape[0]\n",
    "            train_acc += acc\n",
    "        train_losses.append(train_loss / len(train_data))\n",
    "        train_acces.append(train_acc / len(train_data))\n",
    "\n",
    "        test_loss = 0\n",
    "        test_acc = 0\n",
    "        model.eval()\n",
    "        for im_test, label_test in test_data:\n",
    "            # 前向传播\n",
    "            im_test = im_test.cuda()\n",
    "            label_test = label_test.cuda()\n",
    "            label_test = label_test.squeeze()\n",
    "\n",
    "            out_test = model(im_test)\n",
    "            loss_test = criterion(out_test, label_test)\n",
    "\n",
    "            # 记录误差\n",
    "            test_loss += loss_test.item()\n",
    "            # 计算分类的准确率\n",
    "            _, pred_test = torch.max(out_test, 1)\n",
    "            num_correct_test = (pred_test == label_test).sum().item()\n",
    "            acc_test = num_correct_test / im_test.shape[0]\n",
    "            test_acc += acc_test\n",
    "\n",
    "        test_losses.append(test_loss / len(test_data))\n",
    "        test_acces.append(test_acc / len(test_data))\n",
    "\n",
    "        print(\"epoch: %d, train loss: %.6f, train acc: %.6f, test loss: %.6f, test acc: %.6f\" % (\n",
    "            i, train_loss / len(train_data), train_acc / len(train_data), test_loss / len(test_data),\n",
    "            test_acc / len(test_data)))\n",
    "\n",
    "    print(\"train and test-----------------Done!\")\n",
    "\n",
    "    plot_image_train(train_acces, train_losses)\n",
    "    plot_image_test(test_acces, test_losses)\n",
    "\n",
    "    torch.save(model.state_dict(), \"model.pth\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![jupyter](./demo34.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![jupyter](./demo34_2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}