q1k_automated_reports.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate automated session reports for all participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import mne\n",
    "import mne_bids\n",
    "from matplotlib import pyplot as plt\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "import plotly.offline as py\n",
    "import plotly.io as pio\n",
    "pio.renderers.default = \"plotly_mimetype+notebook\"\n",
    "import q1k_init_tools as qit\n",
    "import glob\n",
    "import warnings\n",
    "from nbconvert import HTMLExporter\n",
    "from nbconvert.preprocessors import ExecutePreprocessor\n",
    "from nbformat import read\n",
    "import os\n",
    "import papermill as pm\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Select task parameters and file paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select kind of data experimental or control group\n",
    "\n",
    "dataset_group = \"experimental\"\n",
    "\n",
    "#if dataset_group == \"control\":\n",
    "#    # Control group data\n",
    "#    project_path = \"/home/james/q1k/pilot/q1k-external-pilot/\"\n",
    "#    task_id_in = \"ap\"\n",
    "#    task_id_in_et = \"ap\"\n",
    "#    task_id_out = \"ap\"\n",
    "#    #subject_id = '002'\n",
    "#    session_id = '01'\n",
    "#    run_id = '1'\n",
    "\n",
    "#elif dataset_group == \"experimental\":\n",
    "# Experimental group data\n",
    "\n",
    "project_path = \"/project/def-emayada/q1k/experimental/HSJ/\"\n",
    "init_path = \"code/q1k_eeget_init/\"\n",
    "task_id_in = \"VEP\"\n",
    "task_id_in_et = \"VEP\"\n",
    "task_id_out = \"VEP\"\n",
    "#subject_id = 'Q1K_HSJ_100123_F1'\n",
    "run_id = '1'\n",
    "session_id = '01'\n",
    "site_code = 'HSJ' #'MHC' or 'HSJ'\n",
    "sourcedata_path = \"sourcedata/\" \n",
    "html_reports_path = \"session_reports/\" + task_id_in  + '/'\n",
    "et_sync = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sanity check to see which task you would like to make reports for \n",
    "print(task_id_in)\n",
    "et_sync"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Generate list of session reports already processed so they are not processed again in next step\n",
    "processed_sessions = []\n",
    "\n",
    "for subject in glob.glob(project_path + init_path + \"session_reports/\" + task_id_in + \"/Q1k*.html\"): \n",
    "    processed_sessions.append(subject.split('\\\\')[1][:-5])\n",
    "\n",
    "print('Existing session reports:')\n",
    "processed_sessions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate html session reports for all participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glob.glob(project_path + sourcedata_path + \"eeg/Q1K*/Q1K*\" + task_id_in + '_*.mff')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make sure the output directory exists\n",
    "if not os.path.exists(html_reports_path):\n",
    "    os.makedirs(html_reports_path)\n",
    "\n",
    "# Create a list of sessions with errors\n",
    "error_subjects = []\n",
    "\n",
    "#Loop through existing sessions and execute the q1k_generate_individual_reports.ipynb if a session report does not already exist\n",
    "for file in glob.glob(project_path + sourcedata_path + \"eeg/Q1K*/Q1K*\" + task_id_in + '_*.mff'):\n",
    "    print('Current data file: ' + file)\n",
    "    # Select anything after the Q1K and before the AEP\n",
    "    #subject_id = file.split('_')[2]\n",
    "    site_part = file.split(site_code + \"_\")\n",
    "    subject_number = site_part[1].split(\"_\")[0]\n",
    "    print('Participant number: ' + subject_number)\n",
    "    subject_relation = site_part[1].split(\"_\")[1].split(\"/\")[0]\n",
    "    print('Participant relation: ' + subject_relation)\n",
    "\n",
    "    subject_id_in = subject_number + \"_\" + subject_relation\n",
    "    print('Participant ID input: ' + subject_id_in)\n",
    "    subject_id_out = subject_number.replace('_','').replace('-','') + subject_relation\n",
    "    print('Participant ID output: ' + subject_id_out)\n",
    "    \n",
    "    # Skip sessions that have a session report in the output directory\n",
    "    print(subject_id_in)\n",
    "    if subject_id_in in processed_sessions:\n",
    "        print(subject_id_in + ' has already been processed')\n",
    "        continue    \n",
    "\n",
    "    # Define paths\n",
    "    input_notebook = project_path + init_path +'q1k_generate_individual_reports.ipynb'\n",
    "    print('Input notebook: ' + input_notebook)\n",
    "\n",
    "    # Make sure the directory exists\n",
    "    if not os.path.exists(f'{project_path}{init_path}session_reports/{task_id_in}/executed_notebooks/'):\n",
    "        os.makedirs(f'{project_path}{init_path}session_reports/{task_id_in}/executed_notebooks/')\n",
    "    \n",
    "    output_notebook = f'{project_path}{init_path}{html_reports_path}executed_notebooks/{subject_id_in}_{task_id_in}_executed.ipynb'\n",
    "    print('Output notebook file: ' + output_notebook)\n",
    "    output_html = f'{project_path}{init_path}{html_reports_path}{subject_id_in}_{task_id_in}.html'\n",
    "    print('Output HTML file: ' + output_html)\n",
    "\n",
    "    try:\n",
    "        # Execute the notebook\n",
    "        pm.execute_notebook(input_notebook, output_notebook, kernel_name = 'q1k_env', parameters=dict(subject_id_in=subject_id_in, subject_id_out=subject_id_out,task_id_in=task_id_in, task_id_in_et=task_id_in_et,\n",
    "         task_id_out=task_id_out, run_id=run_id, session_id=session_id, project_path=project_path, dataset_group=dataset_group, site_code=site_code))\n",
    "\n",
    "        # Convert executed notebook to HTML\n",
    "        html_exporter = HTMLExporter()\n",
    "        html_exporter.exclude_input = True\n",
    "\n",
    "        (body, resources) = html_exporter.from_filename(output_notebook)\n",
    "\n",
    "        # Save HTML output\n",
    "        with open(output_html, 'w', encoding='utf-8') as f:\n",
    "            f.write(body)\n",
    "\n",
    "        print(f\"HTML report saved for {subject_id_in}.\")\n",
    "    \n",
    "\n",
    "    except Exception as e:\n",
    "        # Handle the error \n",
    "        error_subjects.append(subject_id_in)\n",
    "        print(f\"Error while processing {subject_id_in}: {e}\")\n",
    "\n",
    "# Print out the list of subjects with errors\n",
    "print( \"These subjects have errors: \" + str(error_subjects) + \" and need to be reprocessed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize all reports at once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for file in glob.glob(\"./../../sourcefiles/EEG/Q1K*/Q1K\" + \"*AEP\" + '*.mff'):\n",
    "#    # Select anything after the Q1K and before the AEP\n",
    "#    subject_id = file.split('\\\\')[1]\n",
    "#    # Skip sessions that have already been processed\n",
    "#    print(subject_id)\n",
    "#    if subject_id in processed_sessions:\n",
    "#        print (subject_id + ' has already been processed')\n",
    "#        continue    \n",
    "\n",
    "#    # Handle participants with error \n",
    "#    try:\n",
    "#        %run -i ./q1k_generate_individual_reports.ipynb --subject_id={subject_id}\n",
    "#    except Exception as e:\n",
    "#    # Handle the error (e.g., log it, skip participant, etc.)\n",
    "#        print(f\"Error while reading raw data for {subject_id}: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "q1k_env",
   "language": "python",
   "name": "q1k_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}