mtYOLO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "89fd8ff0-e8e6-4335-9224-a9f2b93a773b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-31T14:45:55.560201Z",
     "iopub.status.busy": "2024-05-31T14:45:55.560017Z",
     "iopub.status.idle": "2024-05-31T14:45:55.562112Z",
     "shell.execute_reply": "2024-05-31T14:45:55.561848Z",
     "shell.execute_reply.started": "2024-05-31T14:45:55.560188Z"
    }
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "032cba4a-71af-478c-aedf-809e5d9c87ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from ultralytics import YOLO\n",
    "from ultralytics.models.yolo.multi import MultiTaskTrainer\n",
    "import datetime, os, glob\n",
    "\n",
    "model_name='mtyolov8'\n",
    "list_task = ['pose', 'segment', 'multitask']\n",
    "list_model_type = ['', '_ECA'] \n",
    "list_pretrained = ['', '_pretrained'] # Load existing pre-trained YOLO models (Only for pose and segmentation)\n",
    "list_dataset = ['coco', 'cattleeyeview']\n",
    "dir_mtYOLO_root = '/home/user/mtYOLO'\n",
    "epochs = 80\n",
    "patience = 0\n",
    "device = [0, 1]\n",
    "image_size = 640\n",
    "batch_size = 256\n",
    "\n",
    "def time_now():\n",
    "    return str(datetime.datetime.now())\n",
    "    \n",
    "def yolo_train(task, \n",
    "         model_type, \n",
    "         dir_mtYOLO_root, \n",
    "         pretrained='', \n",
    "         loss_type='', \n",
    "         device=[0],\n",
    "         epochs=80, \n",
    "         patience=0, \n",
    "         image_size=640,\n",
    "         batch_size=-1,\n",
    "         model_name='mtyolov8'\n",
    "         ):\n",
    "\n",
    "    ## Check if config files exist\n",
    "    dir_model = f\"{dir_mtYOLO_root}/config/model/{model_name}_{task}_{dataset}{model_type}.yaml\"\n",
    "    dir_data = f\"{dir_mtYOLO_root}/config/dataset/{dataset}_{task}.yaml\"\n",
    "    dir_log = f\"{os.path.dirname(dir_mtYOLO_root)}/logs\"\n",
    "    print(f\"{dir_model} exists: {os.path.exists(dir_model)}\")\n",
    "    print(f\"{dir_data} exists: {os.path.exists(dir_data)}\")\n",
    "\n",
    "    ## Naming configuration\n",
    "    model_name = dir_model.split('/')[-1].split('.')[0]\n",
    "    experiment_name = f'{time_now()[:13]}_{model_name}{pretrained}'\n",
    "    \n",
    "    args = dict(\n",
    "        model=dir_model, #Specifies the model file for training. Accepts a path to either a .pt pretrained model or a .yaml configuration file. Essential for defining the model structure or initializing weights.\n",
    "        data=dir_data, #Path to the dataset configuration file (e.g., coco128.yaml). This file contains dataset-specific parameters, including paths to training and validation data, class names, and number of classes.\n",
    "        project=f'{dir_log}', #None, #Name of the project directory where training outputs are saved. Allows for organized storage of different experiments.\n",
    "        name=f'{experiment_name}', #Name of the training run. Used for creating a subdirectory within the project folder, where training logs and outputs are stored.\n",
    "        exist_ok=True, #If True, allows overwriting of an existing project/name directory. Useful for iterative experimentation without needing to manually clear previous outputs.\n",
    "    \n",
    "        imgsz=image_size, #Target image size for training. All images are resized to this dimension before being fed into the model. Affects model accuracy and computational complexity.\n",
    "        batch=batch_size, #16, #Batch size for training, indicating how many images are processed before the model's internal parameters are updated. AutoBatch (batch=-1) dynamically adjusts the batch size based on GPU memory availability.\n",
    "        epochs=epochs, #Total number of training epochs. Each epoch represents a full pass over the entire dataset. Adjusting this value can affect training duration and model performance.\n",
    "        cache=True, #Enables caching of dataset images in memory (True/ram), on disk (disk), or disables it (False). Improves training speed by reducing disk I/O at the cost of increased memory usage.\n",
    "    #     fraction=1, #Specifies the fraction of the dataset to use for training. Allows for training on a subset of the full dataset, useful for experiments or when resources are limited.\n",
    "    #     seed=0, #Sets the random seed for training, ensuring reproducibility of results across runs with the same configurations.\n",
    "    #     deterministic=False, #Forces deterministic algorithm use, ensuring reproducibility but may affect performance and speed due to the restriction on non-deterministic algorithms.\n",
    "    #     pretrained=True, #Determines whether to start training from a pretrained model. Can be a boolean value or a string path to a specific model from which to load weights. Enhances training efficiency and model performance.\n",
    "    #     resume=False, #Resumes training from the last saved checkpoint. Automatically loads model weights, optimizer state, and epoch count, continuing training seamlessly.\n",
    "    #     freeze=None, #Freezes the first N layers of the model or specified layers by index, reducing the number of trainable parameters. Useful for fine-tuning or transfer learning.\n",
    "    #     time=None, #Maximum training time in hours. If set, this overrides the epochs argument, allowing training to automatically stop after the specified duration. Useful for time-constrained training scenarios.\n",
    "        patience=patience, #Number of epochs to wait without improvement in validation metrics before early stopping the training. Helps prevent overfitting by stopping training when performance plateaus.\n",
    "        verbose=False, #Enables verbose output during training, providing detailed logs and progress updates. Useful for debugging and closely monitoring the training process.\n",
    "    \n",
    "        device=device, #Specifies the computational device(s) for training: a single GPU (device=0), multiple GPUs (device=0,1), CPU (device=cpu), or MPS for Apple silicon (device=mps).\n",
    "        workers=32, #8, #Number of worker threads for data loading (per RANK if Multi-GPU training). Influences the speed of data preprocessing and feeding into the model, especially useful in multi-GPU setups.\n",
    "    \n",
    "        optimizer='auto', # 'AdamW' #Choice of optimizer for training. Options include SGD, Adam, AdamW, NAdam, RAdam, RMSProp etc., or auto for automatic selection based on model configuration. Affects convergence speed and stability.\n",
    "    #     lr0=0.01, #Initial learning rate (i.e. SGD=1E-2, Adam=1E-3) . Adjusting this value is crucial for the optimization process, influencing how rapidly model weights are updated.\n",
    "    #     warmup_epochs=3, #Number of epochs for learning rate warmup, gradually increasing the learning rate from a low value to the initial learning rate to stabilize training early on.\n",
    "    #     warmup_momentum=0.8, #Initial momentum for warmup phase, gradually adjusting to the set momentum over the warmup period.\n",
    "    #     warmup_bias_lr=0.1, #Learning rate for bias parameters during the warmup phase, helping stabilize model training in the initial epochs.\n",
    "    #     lrf=0.01, #Final learning rate as a fraction of the initial rate = (lr0 * lrf), used in conjunction with schedulers to adjust the learning rate over time.\n",
    "    #     cos_lr=False, #Utilizes a cosine learning rate scheduler, adjusting the learning rate following a cosine curve over epochs. Helps in managing learning rate for better convergence.\n",
    "    #     momentum=0.937, #Momentum factor for SGD or beta1 for Adam optimizers, influencing the incorporation of past gradients in the current update.\n",
    "    #     weight_decay=0.0005, #L2 regularization term, penalizing large weights to prevent overfitting.\n",
    "        \n",
    "        # mask_ratio=0, #4, #Downsample ratio for segmentation masks, affecting the resolution of masks used during training.\n",
    "        dropout=0, #Dropout rate for regularization in classification tasks, preventing overfitting by randomly omitting units during training.\n",
    "        \n",
    "    #     single_cls=False, #Treats all classes in multi-class datasets as a single class during training. Useful for binary classification tasks or when focusing on object presence rather than classification.\n",
    "        rect=False, #Enables rectangular training, optimizing batch composition for minimal padding. Can improve efficiency and speed but may affect model accuracy.\n",
    "        \n",
    "        close_mosaic=0, #10, #Disables mosaic data augmentation in the last N epochs to stabilize training before completion. Setting to 0 disables this feature.\n",
    "        # hsv_h=0.015, #0.0-1.0\tAdjusts the hue of the image by a fraction of the color wheel, introducing color variability. Helps the model generalize across different lighting conditions.\n",
    "        # hsv_s=0.7, #0.0-1.0\tAlters the saturation of the image by a fraction, affecting the intensity of colors. Useful for simulating different environmental conditions.\n",
    "        # hsv_v=0.4, #0.0-1.0\tModifies the value (brightness) of the image by a fraction, helping the model to perform well under various lighting conditions.\n",
    "        # degrees=0.0, #-180-+180\tRotates the image randomly within the specified degree range, improving the model's ability to recognize objects at various orientations.\n",
    "        # translate=0.0, #0.0-1.0\tTranslates the image horizontally and vertically by a fraction of the image size, aiding in learning to detect partially visible objects.\n",
    "        # scale=0.0, #>=0.0\tScales the image by a gain factor, simulating objects at different distances from the camera.\n",
    "        # shear=0.0, #-180-+180\tShears the image by a specified degree, mimicking the effect of objects being viewed from different angles.\n",
    "        # perspective=0.0, #0.0-0.001\tApplies a random perspective transformation to the image, enhancing the model's ability to understand objects in 3D space.\n",
    "        # flipud=0.0, #0.0-1.0\tFlips the image upside down with the specified probability, increasing the data variability without affecting the object's characteristics.\n",
    "        # fliplr=0.5, #0.0-1.0\tFlips the image left to right with the specified probability, useful for learning symmetrical objects and increasing dataset diversity.\n",
    "        mosaic=0.0, #0.0-1.0\tCombines four training images into one, simulating different scene compositions and object interactions. Highly effective for complex scene understanding.\n",
    "        # mixup=0.0, #0.0-1.0\tBlends two images and their labels, creating a composite image. Enhances the model's ability to generalize by introducing label noise and visual variability.\n",
    "        # copy_paste=0.0, #0.0-1.0\tCopies objects from one image and pastes them onto another, useful for increasing object instances and learning object occlusion.\n",
    "        # auto_augment='randaugment', # Automatically appliesa predefined augmentation policy (randaugment, autoaugment, augmix), optimizing for classification tasks by diversifying the visual features.\n",
    "        # erasing=0.4, #0.0-1.0\tRandomly erases a portion of the image during classification training, encouraging the model to focus on less obvious features for recognition.\n",
    "        \n",
    "        # box=5,#7.5, #Weight of the box loss component in the loss function, influencing how much emphasis is placed on accurately predicting bounding box coordinates.\n",
    "        # cls=5,#0.5, #Weight of the classification loss in the total loss function, affecting the importance of correct class prediction relative to other components.\n",
    "        # dfl=10,#1.5, #Weight of the distribution focal loss, used in certain YOLO versions for fine-grained classification.\n",
    "        # pose=20,#12, #Weight of the pose loss in models trained for pose estimation, influencing the emphasis on accurately predicting pose keypoints.\n",
    "        # kobj=10,#2, #Weight of the keypoint objectness loss in pose estimation models, balancing detection confidence with pose accuracy.\n",
    "        # nbs=64, #Nominal batch size for normalization of loss.\n",
    "        \n",
    "        # label_smoothing=0, #Applies label smoothing, softening hard labels to a mix of the target label and a uniform distribution over labels, can improve generalization.\n",
    "        overlap_mask=False, #True, #Determines whether segmentation masks should overlap during training, applicable in instance segmentation tasks.\n",
    "    \n",
    "        val=True, #Enables validation during training, allowing for periodic evaluation of model performance on a separate dataset.\n",
    "        plots=True, #Generates and saves plots of training and validation metrics, as well as prediction examples, providing visual insights into model performance and learning progression.\n",
    "        save=True, #Enables saving of training checkpoints and final model weights. Useful for resuming training or model deployment.\n",
    "        save_period=-1, #Frequency of saving model checkpoints, specified in epochs. A value of -1 disables this feature. Useful for saving interim models during long training sessions.\n",
    "    \n",
    "    #     profile=False, #Enables profiling of ONNX and TensorRT speeds during training, useful for optimizing model deployment.\n",
    "    #     amp=True, #Enables Automatic Mixed Precision (AMP) training, reducing memory usage and possibly speeding up training with minimal impact on accuracy.\n",
    "       \n",
    "    )\n",
    "\n",
    "    print(f'{model_name}{model_type}{pretrained} {task} training starts: {time_now()} ')\n",
    "\n",
    "    ## Start training multitask model\n",
    "    if task=='multitask':\n",
    "        trainer = MultiTaskTrainer(overrides=args)\n",
    "        trainer.train()\n",
    "        \n",
    "    else:\n",
    "        ## Load pre-trained YOLO model\n",
    "        if pretrained!='':\n",
    "            ## Change name to load pre-trained YOLO model\n",
    "            if task=='segment':\n",
    "                task_type='seg'\n",
    "            else:\n",
    "                task_type=task\n",
    "            model = YOLO(dir_model).load(f'yolov8n-{task_type}.pt')\n",
    "        \n",
    "        else:\n",
    "            task_type=task\n",
    "            model = YOLO(dir_model)\n",
    "            \n",
    "        model.train(**args)\n",
    "\n",
    "    print(f'{model_name}{model_type}{pretrained} {task} training ends: {time_now()} \\n')\n",
    "\n",
    "for dataset in list_dataset:\n",
    "    for task in list_task:\n",
    "        for model_type in list_model_type:\n",
    "            for pretrained in list_pretrained:\n",
    "                if (task=='multitask') and (pretrained!=''):\n",
    "                    None\n",
    "                else:\n",
    "                    print(f'Dataset: {dataset}, Task: {task}, Model: {model_type}{pretrained}')\n",
    "\n",
    "                    ## Start training\n",
    "                    yolo_train(task=task, \n",
    "                               model_type=model_type, \n",
    "                               dir_mtYOLO_root=dir_mtYOLO_root, \n",
    "                               pretrained=pretrained,\n",
    "                               device=device,\n",
    "                               epochs=epochs, \n",
    "                               patience=patience, \n",
    "                               image_size=image_size,\n",
    "                               batch_size=batch_size,\n",
    "                               model_name=model_name\n",
    "                              )\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50624f70-14b1-4798-b5b1-76022b492226",
   "metadata": {},
   "source": [
    "# Validation / Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee41f716-3c6f-43c8-8165-fd4a38cba055",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Validation and Prediction\n",
    "import torch\n",
    "# from ultralytics import YOLO\n",
    "# from ultralytics.models.yolo.multi import MultiTaskPredictor, MultiTaskValidator\n",
    "import datetime, os, glob\n",
    "\n",
    "model_mode = 'val' #['val', 'predict']\n",
    "list_task = ['pose', 'segment', 'multitask']\n",
    "list_model_type = ['', '_ECA'] \n",
    "list_pretrained = ['', '_pretrained'] # Load existing pre-trained YOLO models (Only for pose and segmentation)\n",
    "list_dataset = ['coco', 'cattleeyeview']\n",
    "dir_log_root = '/home/user/mtYOLO/log' \n",
    "dir_image = '/home/user/mtYOLO/data/multitask/images/val/'\n",
    "\n",
    "device = [0, 1]\n",
    "image_size = 640\n",
    "batch_size = 256\n",
    "\n",
    "\n",
    "for dataset in list_dataset[:1]:\n",
    "    for task in list_task[:1]:\n",
    "        for model_type in list_model_type[:1]:\n",
    "            for pretrained in list_pretrained[:1]:\n",
    "                dir_model_checkpoint = max([file for file in sorted(glob.iglob(dir_log_root + f'/**/*{task}*{model_type}*{pretrained}*/**/best.pt', recursive=True))])\n",
    "                print(f'Task: {task}, Model: YOLOv8{model_type}{pretrained}, Directory of latest checkpoint: {dir_model_checkpoint}')\n",
    "                print(f\"{dir_image} exists: {os.path.exists(dir_image)}\")\n",
    "\n",
    "                ## Configuration\n",
    "                args = dict(\n",
    "                    model=dir_model_checkpoint, #Specifies the model file for training. Accepts a path to either a .pt pretrained model or a .yaml configuration file. Essential for defining the model structure or initializing weights.\n",
    "                    )\n",
    "\n",
    "                ## Load model\n",
    "                model = YOLO(**args)\n",
    "                \n",
    "                ## Print model information\n",
    "                model.info()\n",
    "\n",
    "                if model_mode=='val':\n",
    "                    ## Validation\n",
    "                    model.val(\n",
    "                        data = dir_image, #None, # Specifies the path to the dataset configuration file (e.g., coco8.yaml). This file includes paths to validation data, class names, and number of classes.\n",
    "                        imgsz = image_size, #640, # Defines the size of input images. All images are resized to this dimension before processing.\n",
    "                        # batch = 16, # Sets the number of images per batch. Use -1 for AutoBatch, which automatically adjusts based on GPU memory availability.\n",
    "                        # save_json = False, # If True, saves the results to a JSON file for further analysis or integration with other tools.\n",
    "                        # save_hybrid = True, #False, # If True, saves a hybrid version of labels that combines original annotations with additional model predictions.\n",
    "                        # conf = 0.001, # Sets the minimum confidence threshold for detections. Detections with confidence below this threshold are discarded.\n",
    "                        # iou = 0.6, # Sets the Intersection Over Union (IoU) threshold for Non-Maximum Suppression (NMS). Helps in reducing duplicate detections.\n",
    "                        # max_det = 300, # Limits the maximum number of detections per image. Useful in dense scenes to prevent excessive detections.\n",
    "                        # half = True, # Enables half-precision (FP16) computation, reducing memory usage and potentially increasing speed with minimal impact on accuracy.\n",
    "                        device = device, #None, # Specifies the device for validation (cpu, cuda:0, etc.). Allows flexibility in utilizing CPU or GPU resources.\n",
    "                        # dnn = False, # If True, uses the OpenCV DNN module for ONNX model inference, offering an alternative to PyTorch inference methods.\n",
    "                        plots = True, # False, # When set to True, generates and saves plots of predictions versus ground truth for visual evaluation of the model's performance.\n",
    "                        # rect = False, # If True, uses rectangular inference for batching, reducing padding and potentially increasing speed and efficiency.\n",
    "                        # split = 'val' # Determines the dataset split to use for validation (val, test, or train). Allows flexibility in choosing the data segment for performance evaluation.\n",
    "                    )\n",
    "\n",
    "                if model_mode=='predict':\n",
    "                    ## Predict\n",
    "                    model.predict(\n",
    "                        source = dir_image, #\tSpecifies the data source for inference. Can be an image path, video file, directory, URL, or device ID for live feeds. Supports a wide range of formats and sources, enabling flexible application across different types of input.\n",
    "                        # conf = 0.25, # Sets the minimum confidence threshold for detections. Objects detected with confidence below this threshold will be disregarded. Adjusting this value can help reduce false positives.\n",
    "                        # iou = 0.7, # Intersection Over Union (IoU) threshold for Non-Maximum Suppression (NMS). Lower values result in fewer detections by eliminating overlapping boxes, useful for reducing duplicates.\n",
    "                        # imgsz = 640, # Defines the image size for inference. Can be a single integer 640 for square resizing or a (height, width) tuple. Proper sizing can improve detection accuracy and processing speed.\n",
    "                        # half = False, # Enables half-precision (FP16) inference, which can speed up model inference on supported GPUs with minimal impact on accuracy.\n",
    "                        # device = None, # Specifies the device for inference (e.g., cpu, cuda:0 or 0). Allows users to select between CPU, a specific GPU, or other compute devices for model execution.\n",
    "                        # max_det = 300, # Maximum number of detections allowed per image. Limits the total number of objects the model can detect in a single inference, preventing excessive outputs in dense scenes.\n",
    "                        # vid_stride = 1, # Frame stride for video inputs. Allows skipping frames in videos to speed up processing at the cost of temporal resolution. A value of 1 processes every frame, higher values skip frames.\n",
    "                        # stream_buffer = False, # Determines if all frames should be buffered when processing video streams (True), or if the model should return the most recent frame (False). Useful for real-time applications.\n",
    "                        # visualize = True, #False, # Activates visualization of model features during inference, providing insights into what the model is \"seeing\". Useful for debugging and model interpretation.\n",
    "                        # augment = False, # Enables test-time augmentation (TTA) for predictions, potentially improving detection robustness at the cost of inference speed.\n",
    "                        # agnostic_nms = False, # Enables class-agnostic Non-Maximum Suppression (NMS), which merges overlapping boxes of different classes. Useful in multi-class detection scenarios where class overlap is common.\n",
    "                        # classes = None, # Filters predictions to a set of class IDs. Only detections belonging to the specified classes will be returned. Useful for focusing on relevant objects in multi-class detection tasks.\n",
    "                        # retina_masks = False, # Uses high-resolution segmentation masks if available in the model. This can enhance mask quality for segmentation tasks, providing finer detail.\n",
    "                        # embed = None, # Specifies the layers from which to extract feature vectors or embeddings. Useful for downstream tasks like clustering or similarity search.\n",
    "                                        \n",
    "                        # show = False, # If True, displays the annotated images or videos in a window. Useful for immediate visual feedback during development or testing.\n",
    "                        # save = False, # Enables saving of the annotated images or videos to file. Useful for documentation, further analysis, or sharing results.\n",
    "                        # save_frames = False, # When processing videos, saves individual frames as images. Useful for extracting specific frames or for detailed frame-by-frame analysis.\n",
    "                        # save_txt = False, # Saves detection results in a text file, following the format [class] [x_center] [y_center] [width] [height] [confidence]. Useful for integration with other analysis tools.\n",
    "                        # save_conf = False, # Includes confidence scores in the saved text files. Enhances the detail available for post-processing and analysis.\n",
    "                        # save_crop = False, # Saves cropped images of detections. Useful for dataset augmentation, analysis, or creating focused datasets for specific objects.\n",
    "                        # show_labels = True, # Displays labels for each detection in the visual output. Provides immediate understanding of detected objects.\n",
    "                        # show_conf = True, # Displays the confidence score for each detection alongside the label. Gives insight into the model's certainty for each detection.\n",
    "                        # show_boxes = True, # Draws bounding boxes around detected objects. Essential for visual identification and location of objects in images or video frames.\n",
    "                        # line_width = None\t# Specifies the line width of bounding boxes. If None, the line width is automatically adjusted based on the image size. Provides visual customization for clarity.\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06ce4dcf-8836-4559-ac07-2e1168ef0934",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mtyolo",
   "language": "python",
   "name": "mtyolo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}