attribution_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-13T16:29:27.855869Z",
     "start_time": "2024-09-13T16:29:27.781372Z"
    }
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"all_results.csv\")\n",
    "# fix up slightly broken csv\n",
    "df['train_col_sd'] = df['model'].str.extract(\"col_sd_(\\d.0)\")\n",
    "df.train_col_sd = df.train_col_sd.astype(float)\n",
    "\n",
    "df['additive_importance'] = df['attribution_incorrect.blackbg_withpix_model_accuracy'] - df['attribution_incorrect.blackbg_withattrpix_model_accuracy']\n",
    "\n",
    "df['subtractive_importance'] = df['attribution_incorrect.truebg_withblackattrpix_model_accuracy'] - df['attribution_incorrect.truebg_withblackpix_model_accuracy']\n",
    "\n",
    "# Extra columns to help later\n",
    "nice_names = {\"predict_importance_absgrad\": \"grad. mag.\",\n",
    "              \"predict_importance_grad_inp\": \"grad. $\\\\times$ input\",\n",
    "              \"predict_importance_integrated_gradient\": \"integrated (b)\",\n",
    "              \"predict_importance_integrated_gradient_white\": \"integrated (w)\",\n",
    "              }\n",
    "df[\"Attribution method\"] = df[\"attribution_method\"].map(nice_names)\n",
    "\n",
    "df[\"\\\\textbf{p}redicted/\\\\textbf{t}rue\"] = df[\"use_y_true\"].map(lambda x: \"t\" if x else \"p\")\n",
    "df[\"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\"] = df[\"use_softmax\"].map(lambda x: \"s\" if x else \"l\")\n",
    "\n",
    "for c in df.columns:\n",
    "    if \"accuracy\" in c:\n",
    "        df[c] = df[c].map(lambda x: x*100.)\n",
    "\n",
    "df2 = df[(df['dataset'] == \"SinglePixelDataset\") & (df['train_pos_sd'] == 0) & (df['test_pos_sd'] == 0) & (df['test_col_sd'] == 0) & (df['train_col_sd'] == 0) & (df[\"data_seed\"] == 1) & (df[\"attribution_method\"] != \"predict_importance_absgrad_absinp\") & \n",
    " (df[\"model_type\"] == \"resnet18_3x3\") & \n",
    " (df[\"use_softmax\"] == True) &\n",
    " (df[\"use_y_true\"] == False) \n",
    "]\n",
    "df2#[['attribution_incorrect.truebg_withblackpix_model_accuracy', 'model']]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "      overall.withpix_model_accuracy  overall.attribution_correct_accuracy  \\\n",
       "800                            99.82                                 99.17   \n",
       "804                            99.82                                 61.87   \n",
       "812                            99.82                                 99.96   \n",
       "816                            99.82                                 97.34   \n",
       "2900                           99.84                                 98.95   \n",
       "2904                           99.84                                 62.19   \n",
       "2912                           99.84                                 99.51   \n",
       "2916                           99.84                                 99.82   \n",
       "5000                           99.74                                 99.23   \n",
       "5004                           99.74                                 64.02   \n",
       "5012                           99.74                                 95.63   \n",
       "5016                           99.74                                 98.03   \n",
       "\n",
       "      overall.nopix_model_accuracy  \\\n",
       "800                          13.76   \n",
       "804                          13.76   \n",
       "812                          13.76   \n",
       "816                          13.76   \n",
       "2900                         13.04   \n",
       "2904                         13.04   \n",
       "2912                         13.04   \n",
       "2916                         13.04   \n",
       "5000                         10.49   \n",
       "5004                         10.49   \n",
       "5012                         10.49   \n",
       "5016                         10.49   \n",
       "\n",
       "      overall.truebg_withblackpix_model_accuracy  \\\n",
       "800                                        13.16   \n",
       "804                                        13.16   \n",
       "812                                        13.16   \n",
       "816                                        13.16   \n",
       "2900                                       14.49   \n",
       "2904                                       14.49   \n",
       "2912                                       14.49   \n",
       "2916                                       14.49   \n",
       "5000                                       10.34   \n",
       "5004                                       10.34   \n",
       "5012                                       10.34   \n",
       "5016                                       10.34   \n",
       "\n",
       "      attribution_incorrect.nopix_model_accuracy  \\\n",
       "800                                    12.048193   \n",
       "804                                    16.338841   \n",
       "812                                    25.000000   \n",
       "816                                    92.857143   \n",
       "2900                                   19.047619   \n",
       "2904                                   14.890241   \n",
       "2912                                   93.877551   \n",
       "2916                                   77.777778   \n",
       "5000                                   10.389610   \n",
       "5004                                   14.591440   \n",
       "5012                                   98.855835   \n",
       "5016                                   95.939086   \n",
       "\n",
       "      attribution_incorrect.withpix_model_accuracy  \\\n",
       "800                                     100.000000   \n",
       "804                                      99.868870   \n",
       "812                                      25.000000   \n",
       "816                                      99.248120   \n",
       "2900                                    100.000000   \n",
       "2904                                     99.709072   \n",
       "2912                                     97.959184   \n",
       "2916                                     94.444444   \n",
       "5000                                     98.701299   \n",
       "5004                                     99.444136   \n",
       "5012                                     98.855835   \n",
       "5016                                     96.446701   \n",
       "\n",
       "      attribution_incorrect.blackbg_withpix_model_accuracy  \\\n",
       "800                                          100.000000      \n",
       "804                                          100.000000      \n",
       "812                                          100.000000      \n",
       "816                                          100.000000      \n",
       "2900                                         100.000000      \n",
       "2904                                         100.000000      \n",
       "2912                                         100.000000      \n",
       "2916                                         100.000000      \n",
       "5000                                         100.000000      \n",
       "5004                                          96.275709      \n",
       "5012                                         100.000000      \n",
       "5016                                         100.000000      \n",
       "\n",
       "      attribution_incorrect.blackbg_withattrpix_model_accuracy  \\\n",
       "800                                            8.433735          \n",
       "804                                           11.591922          \n",
       "812                                           75.000000          \n",
       "816                                          100.000000          \n",
       "2900                                          12.380952          \n",
       "2904                                          21.237768          \n",
       "2912                                          22.448980          \n",
       "2916                                          83.333333          \n",
       "5000                                           7.792208          \n",
       "5004                                          14.174541          \n",
       "5012                                          89.016018          \n",
       "5016                                          95.939086          \n",
       "\n",
       "      attribution_incorrect.truebg_withblackattrpix_model_accuracy  \\\n",
       "800                                           98.795181              \n",
       "804                                           99.108314              \n",
       "812                                            0.000000              \n",
       "816                                           98.872180              \n",
       "2900                                          99.047619              \n",
       "2904                                          98.492462              \n",
       "2912                                          95.918367              \n",
       "2916                                          83.333333              \n",
       "5000                                          97.402597              \n",
       "5004                                          98.999444              \n",
       "5012                                          98.627002              \n",
       "5016                                          96.446701              \n",
       "\n",
       "      attribution_incorrect.truebg_withblackpix_model_accuracy  ...  \\\n",
       "800                                           12.048193         ...   \n",
       "804                                           14.975085         ...   \n",
       "812                                            0.000000         ...   \n",
       "816                                           93.609023         ...   \n",
       "2900                                          16.190476         ...   \n",
       "2904                                          21.660936         ...   \n",
       "2912                                          83.673469         ...   \n",
       "2916                                          72.222222         ...   \n",
       "5000                                           7.792208         ...   \n",
       "5004                                          14.452474         ...   \n",
       "5012                                          98.169336         ...   \n",
       "5016                                          95.431472         ...   \n",
       "\n",
       "      test_col_sd                            attribution_method  use_y_true  \\\n",
       "800           0.0                    predict_importance_absgrad       False   \n",
       "804           0.0                   predict_importance_grad_inp       False   \n",
       "812           0.0        predict_importance_integrated_gradient       False   \n",
       "816           0.0  predict_importance_integrated_gradient_white       False   \n",
       "2900          0.0                    predict_importance_absgrad       False   \n",
       "2904          0.0                   predict_importance_grad_inp       False   \n",
       "2912          0.0        predict_importance_integrated_gradient       False   \n",
       "2916          0.0  predict_importance_integrated_gradient_white       False   \n",
       "5000          0.0                    predict_importance_absgrad       False   \n",
       "5004          0.0                   predict_importance_grad_inp       False   \n",
       "5012          0.0        predict_importance_integrated_gradient       False   \n",
       "5016          0.0  predict_importance_integrated_gradient_white       False   \n",
       "\n",
       "     use_softmax                                              model  \\\n",
       "800         True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "804         True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "812         True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "816         True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "2900        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "2904        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "2912        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "2916        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "5000        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "5004        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "5012        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "5016        True  /scratch/jsh2/single-pixel-runs/resnet18_3x3-3...   \n",
       "\n",
       "     additive_importance  subtractive_importance    Attribution method  \\\n",
       "800             0.915663                0.867470            grad. mag.   \n",
       "804             0.884081                0.841332  grad. $\\times$ input   \n",
       "812             0.250000                0.000000        integrated (b)   \n",
       "816             0.000000                0.052632        integrated (w)   \n",
       "2900            0.876190                0.828571            grad. mag.   \n",
       "2904            0.787622                0.768315  grad. $\\times$ input   \n",
       "2912            0.775510                0.122449        integrated (b)   \n",
       "2916            0.166667                0.111111        integrated (w)   \n",
       "5000            0.922078                0.896104            grad. mag.   \n",
       "5004            0.821012                0.845470  grad. $\\times$ input   \n",
       "5012            0.109840                0.004577        integrated (b)   \n",
       "5016            0.040609                0.010152        integrated (w)   \n",
       "\n",
       "      \\textbf{p}redicted/\\textbf{t}rue  \\textbf{s}oftmax/\\textbf{l}ogit  \n",
       "800                                  p                                s  \n",
       "804                                  p                                s  \n",
       "812                                  p                                s  \n",
       "816                                  p                                s  \n",
       "2900                                 p                                s  \n",
       "2904                                 p                                s  \n",
       "2912                                 p                                s  \n",
       "2916                                 p                                s  \n",
       "5000                                 p                                s  \n",
       "5004                                 p                                s  \n",
       "5012                                 p                                s  \n",
       "5016                                 p                                s  \n",
       "\n",
       "[12 rows x 30 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall.withpix_model_accuracy</th>\n",
       "      <th>overall.attribution_correct_accuracy</th>\n",
       "      <th>overall.nopix_model_accuracy</th>\n",
       "      <th>overall.truebg_withblackpix_model_accuracy</th>\n",
       "      <th>attribution_incorrect.nopix_model_accuracy</th>\n",
       "      <th>attribution_incorrect.withpix_model_accuracy</th>\n",
       "      <th>attribution_incorrect.blackbg_withpix_model_accuracy</th>\n",
       "      <th>attribution_incorrect.blackbg_withattrpix_model_accuracy</th>\n",
       "      <th>attribution_incorrect.truebg_withblackattrpix_model_accuracy</th>\n",
       "      <th>attribution_incorrect.truebg_withblackpix_model_accuracy</th>\n",
       "      <th>...</th>\n",
       "      <th>test_col_sd</th>\n",
       "      <th>attribution_method</th>\n",
       "      <th>use_y_true</th>\n",
       "      <th>use_softmax</th>\n",
       "      <th>model</th>\n",
       "      <th>additive_importance</th>\n",
       "      <th>subtractive_importance</th>\n",
       "      <th>Attribution method</th>\n",
       "      <th>\\textbf{p}redicted/\\textbf{t}rue</th>\n",
       "      <th>\\textbf{s}oftmax/\\textbf{l}ogit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>99.82</td>\n",
       "      <td>99.17</td>\n",
       "      <td>13.76</td>\n",
       "      <td>13.16</td>\n",
       "      <td>12.048193</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>8.433735</td>\n",
       "      <td>98.795181</td>\n",
       "      <td>12.048193</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_absgrad</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.915663</td>\n",
       "      <td>0.867470</td>\n",
       "      <td>grad. mag.</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>99.82</td>\n",
       "      <td>61.87</td>\n",
       "      <td>13.76</td>\n",
       "      <td>13.16</td>\n",
       "      <td>16.338841</td>\n",
       "      <td>99.868870</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>11.591922</td>\n",
       "      <td>99.108314</td>\n",
       "      <td>14.975085</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_grad_inp</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.884081</td>\n",
       "      <td>0.841332</td>\n",
       "      <td>grad. $\\times$ input</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>812</th>\n",
       "      <td>99.82</td>\n",
       "      <td>99.96</td>\n",
       "      <td>13.76</td>\n",
       "      <td>13.16</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_integrated_gradient</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>integrated (b)</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>99.82</td>\n",
       "      <td>97.34</td>\n",
       "      <td>13.76</td>\n",
       "      <td>13.16</td>\n",
       "      <td>92.857143</td>\n",
       "      <td>99.248120</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.872180</td>\n",
       "      <td>93.609023</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_integrated_gradient_white</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.052632</td>\n",
       "      <td>integrated (w)</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2900</th>\n",
       "      <td>99.84</td>\n",
       "      <td>98.95</td>\n",
       "      <td>13.04</td>\n",
       "      <td>14.49</td>\n",
       "      <td>19.047619</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>12.380952</td>\n",
       "      <td>99.047619</td>\n",
       "      <td>16.190476</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_absgrad</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.876190</td>\n",
       "      <td>0.828571</td>\n",
       "      <td>grad. mag.</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2904</th>\n",
       "      <td>99.84</td>\n",
       "      <td>62.19</td>\n",
       "      <td>13.04</td>\n",
       "      <td>14.49</td>\n",
       "      <td>14.890241</td>\n",
       "      <td>99.709072</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>21.237768</td>\n",
       "      <td>98.492462</td>\n",
       "      <td>21.660936</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_grad_inp</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.787622</td>\n",
       "      <td>0.768315</td>\n",
       "      <td>grad. $\\times$ input</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2912</th>\n",
       "      <td>99.84</td>\n",
       "      <td>99.51</td>\n",
       "      <td>13.04</td>\n",
       "      <td>14.49</td>\n",
       "      <td>93.877551</td>\n",
       "      <td>97.959184</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>22.448980</td>\n",
       "      <td>95.918367</td>\n",
       "      <td>83.673469</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_integrated_gradient</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.775510</td>\n",
       "      <td>0.122449</td>\n",
       "      <td>integrated (b)</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916</th>\n",
       "      <td>99.84</td>\n",
       "      <td>99.82</td>\n",
       "      <td>13.04</td>\n",
       "      <td>14.49</td>\n",
       "      <td>77.777778</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>83.333333</td>\n",
       "      <td>83.333333</td>\n",
       "      <td>72.222222</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_integrated_gradient_white</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>integrated (w)</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5000</th>\n",
       "      <td>99.74</td>\n",
       "      <td>99.23</td>\n",
       "      <td>10.49</td>\n",
       "      <td>10.34</td>\n",
       "      <td>10.389610</td>\n",
       "      <td>98.701299</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>7.792208</td>\n",
       "      <td>97.402597</td>\n",
       "      <td>7.792208</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_absgrad</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.922078</td>\n",
       "      <td>0.896104</td>\n",
       "      <td>grad. mag.</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5004</th>\n",
       "      <td>99.74</td>\n",
       "      <td>64.02</td>\n",
       "      <td>10.49</td>\n",
       "      <td>10.34</td>\n",
       "      <td>14.591440</td>\n",
       "      <td>99.444136</td>\n",
       "      <td>96.275709</td>\n",
       "      <td>14.174541</td>\n",
       "      <td>98.999444</td>\n",
       "      <td>14.452474</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_grad_inp</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.821012</td>\n",
       "      <td>0.845470</td>\n",
       "      <td>grad. $\\times$ input</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5012</th>\n",
       "      <td>99.74</td>\n",
       "      <td>95.63</td>\n",
       "      <td>10.49</td>\n",
       "      <td>10.34</td>\n",
       "      <td>98.855835</td>\n",
       "      <td>98.855835</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>89.016018</td>\n",
       "      <td>98.627002</td>\n",
       "      <td>98.169336</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_integrated_gradient</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.109840</td>\n",
       "      <td>0.004577</td>\n",
       "      <td>integrated (b)</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5016</th>\n",
       "      <td>99.74</td>\n",
       "      <td>98.03</td>\n",
       "      <td>10.49</td>\n",
       "      <td>10.34</td>\n",
       "      <td>95.939086</td>\n",
       "      <td>96.446701</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>95.939086</td>\n",
       "      <td>96.446701</td>\n",
       "      <td>95.431472</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>predict_importance_integrated_gradient_white</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>/scratch/jsh2/single-pixel-runs/resnet18_3x3-3...</td>\n",
       "      <td>0.040609</td>\n",
       "      <td>0.010152</td>\n",
       "      <td>integrated (w)</td>\n",
       "      <td>p</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 30 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 310
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-13T16:18:38.391394Z",
     "start_time": "2024-09-13T16:18:38.389324Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from functools import partial\n",
    "\n",
    "\n",
    "def format_acc(row, key, inc_std=True):\n",
    "    mean = row[(key, 'mean')]\n",
    "    if not inc_std:\n",
    "        return f\"{mean:.1f}%\"\n",
    "    \n",
    "    std = row[(key, 'std')]\n",
    "    \n",
    "    if std != std:\n",
    "        return f\"{mean:.1f}% (tbc)\"\n",
    "    return f\"{mean:.1f}% ({std:.1f})\""
   ],
   "id": "e509dbd09bb59a0f",
   "outputs": [],
   "execution_count": 297
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-13T16:18:38.507747Z",
     "start_time": "2024-09-13T16:18:38.490557Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df2 = df[(df['dataset'] == \"SinglePixelDataset\") & (df['train_pos_sd'] == 0) & (df['test_pos_sd'] == 0) & (df['test_col_sd'] == 0) & (df['train_col_sd'] == 0) & (df[\"data_seed\"] == 1) & (df[\"attribution_method\"] != \"predict_importance_absgrad_absinp\") & \n",
    " (df[\"model_type\"] == \"resnet18_3x3\") & \n",
    " (df[\"use_softmax\"] == True) &\n",
    " (df[\"use_y_true\"] == False) \n",
    "]\n",
    "\n",
    "table1 = df2.groupby([\"dataset\", \"data_seed\", \"train_pos_sd\", \"train_col_sd\", \"model_type\", \"attribution_method\", \"use_y_true\", \"use_softmax\", \"test_col_sd\", \"test_pos_sd\", \"Attribution method\", \"\\\\textbf{p}redicted/\\\\textbf{t}rue\",\n",
    "    \"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\"], as_index=False).agg({\n",
    "    \"overall.withpix_model_accuracy\": ['mean', 'std'],\n",
    "    \"overall.nopix_model_accuracy\": ['mean', 'std'],\n",
    "    \"overall.attribution_correct_accuracy\": ['mean', 'std']},)\n",
    "\n",
    "table1[('Val. acc.', '')] = table1.apply(partial(format_acc, key=\"overall.withpix_model_accuracy\"), axis=1)\n",
    "table1[('No pixel val. acc.', '')] = table1.apply(partial(format_acc, key=\"overall.nopix_model_accuracy\"), axis=1)\n",
    "table1[('Pointing acc.', '')] = table1.apply(partial(format_acc, key=\"overall.attribution_correct_accuracy\"), axis=1)\n",
    "\n",
    "table1 = table1.sort_values([\"attribution_method\", \"model_type\"])\n",
    "\n",
    "latex = table1.to_latex(columns=[\n",
    "    (\"Attribution method\", ''),\n",
    "    (\"Val. acc.\", ''),\n",
    "    (\"No pixel val. acc.\", ''),\n",
    "    (\"Pointing acc.\", '')\n",
    "    ],\n",
    "    float_format='%.3f',\n",
    "    caption=\"Pointing accuracy of different attribution methods. Three models of each type (resnet18 and vgg16) were trained, and results show the mean (and std. dev.) of performance. All models learned to solve the task (near-perfect val. acc.), relying heavily on the discriminative pixel (hence low `No pixel val. acc.'). \",\n",
    "    label=\"tab:pointing_acc\",\n",
    "    index=False\n",
    "    )\n",
    "print(latex.replace(\"model_type\", \"Model\").replace(\"resnet18_3x3\", \"resnet18\").replace(\"begin{table}\", \"begin{table}[h]\\n\\\\centering\").replace(\"acc. &\",\"&\").replace(\"acc. \\\\\\\\\",\"\\\\\\\\\").replace(\"\\\\textbf{t}rue\",\"\").replace(\"\\\\textbf{l}ogit\",\"\").replace(\"method &\",\"&\").replace(\"&  &  &  &  &  &  \\\\\\\\\",\"&  method & \\\\textbf{t}rue  & \\\\textbf{l}ogit & acc. & acc. & acc. \\\\\\\\\").replace(\"%\", \"\\\\%\"))"
   ],
   "id": "11167ba51a72ca28",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}[h]\n",
      "\\centering\n",
      "\\caption{Pointing accuracy of different attribution methods. Three models of each type (resnet18 and vgg16) were trained, and results show the mean (and std. dev.) of performance. All models learned to solve the task (near-perfect val. acc.), relying heavily on the discriminative pixel (hence low `No pixel val. acc.'). }\n",
      "\\label{tab:pointing_acc}\n",
      "\\begin{tabular}{llll}\n",
      "\\toprule\n",
      "Attribution & Val. & No pixel val. & Pointing \\\\\n",
      " &  &  &  \\\\\n",
      "\\midrule\n",
      "grad. mag. & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.1\\% (0.1) \\\\\n",
      "grad. $\\times$ input & 99.8\\% (0.1) & 12.4\\% (1.7) & 62.7\\% (1.2) \\\\\n",
      "integrated (b) & 99.8\\% (0.1) & 12.4\\% (1.7) & 98.4\\% (2.4) \\\\\n",
      "integrated (w) & 99.8\\% (0.1) & 12.4\\% (1.7) & 98.4\\% (1.3) \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 298
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-13T16:18:38.651004Z",
     "start_time": "2024-09-13T16:18:38.627315Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df2 = df[(df['dataset'] == \"SinglePixelDataset\") & (df['train_pos_sd'] == 0) & (df['test_pos_sd'] == 0) & (df['test_col_sd'] == 0) & (df['train_col_sd'] == 0) & (df[\"data_seed\"] == 1) & (df[\"attribution_method\"] != \"predict_importance_absgrad_absinp\")]\n",
    "\n",
    "table1 = df2.groupby([\"dataset\", \"data_seed\", \"train_pos_sd\", \"train_col_sd\", \"model_type\", \"attribution_method\", \"use_y_true\", \"use_softmax\", \"test_col_sd\", \"test_pos_sd\", \"Attribution method\", \"\\\\textbf{p}redicted/\\\\textbf{t}rue\",\n",
    "    \"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\"], as_index=False).agg({\n",
    "    \"overall.withpix_model_accuracy\": ['mean', 'std'],\n",
    "    \"overall.nopix_model_accuracy\": ['mean', 'std'],\n",
    "    \"overall.attribution_correct_accuracy\": ['mean', 'std']},)\n",
    "\n",
    "table1[('Val. acc.', '')] = table1.apply(partial(format_acc, key=\"overall.withpix_model_accuracy\"), axis=1)\n",
    "table1[('No pixel val. acc.', '')] = table1.apply(partial(format_acc, key=\"overall.nopix_model_accuracy\"), axis=1)\n",
    "table1[('Pointing acc.', '')] = table1.apply(partial(format_acc, key=\"overall.attribution_correct_accuracy\"), axis=1)\n",
    "\n",
    "table1 = table1.sort_values([\"attribution_method\", \"model_type\"])\n",
    "\n",
    "latex = table1.to_latex(columns=[\n",
    "    (\"Attribution method\", ''),\n",
    "    (\"model_type\", ''),\n",
    "    (\"\\\\textbf{p}redicted/\\\\textbf{t}rue\", ''),\n",
    "    (\"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\", ''),\n",
    "    (\"Val. acc.\", ''),\n",
    "    (\"No pixel val. acc.\", ''),\n",
    "    (\"Pointing acc.\", '')\n",
    "    ],\n",
    "    float_format='%.3f',\n",
    "    caption=\"Pointing accuracy of different attribution methods. Three models of each type (resnet18 and vgg16) were trained, and results show the mean (and std. dev.) of performance. All models learned to solve the task (near-perfect val. acc.), relying heavily on the discriminative pixel (hence low `No pixel val. acc.'). \",\n",
    "    label=\"tab:pointing_acc\",\n",
    "    index=False\n",
    "    )\n",
    "print(latex.replace(\"model_type\", \"Model\").replace(\"resnet18_3x3\", \"resnet18\").replace(\"begin{table}\", \"begin{table}[H]\\n\\\\centering\").replace(\"acc. &\",\"&\").replace(\"acc. \\\\\\\\\",\"\\\\\\\\\").replace(\"\\\\textbf{t}rue\",\"\").replace(\"\\\\textbf{l}ogit\",\"\").replace(\"method &\",\"&\").replace(\"&  &  &  &  &  &  \\\\\\\\\",\"&  method & \\\\textbf{t}rue  & \\\\textbf{l}ogit & acc. & acc. & acc. \\\\\\\\\").replace(\"%\", \"\\\\%\"))"
   ],
   "id": "a2fca02cbecc7a89",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}[H]\n",
      "\\centering\n",
      "\\caption{Pointing accuracy of different attribution methods. Three models of each type (resnet18 and vgg16) were trained, and results show the mean (and std. dev.) of performance. All models learned to solve the task (near-perfect val. acc.), relying heavily on the discriminative pixel (hence low `No pixel val. acc.'). }\n",
      "\\label{tab:pointing_acc}\n",
      "\\begin{tabular}{lllllll}\n",
      "\\toprule\n",
      "Model & Attribution & \\textbf{p}redicted/ & \\textbf{s}oftmax/ & Val. & No pixel val. & Pointing \\\\\n",
      " &  method & \\textbf{t}rue  & \\textbf{l}ogit & acc. & acc. & acc. \\\\\n",
      "\\midrule\n",
      "resnet18 & grad. mag. & p & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 97.7\\% (0.6) \\\\\n",
      "resnet18 & grad. mag. & p & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.1\\% (0.1) \\\\\n",
      "resnet18 & grad. mag. & t & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 97.7\\% (0.6) \\\\\n",
      "resnet18 & grad. mag. & t & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.1\\% (0.1) \\\\\n",
      "vgg16 & grad. mag. & p & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & grad. mag. & p & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & grad. mag. & t & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & grad. mag. & t & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "resnet18 & grad. $\\times$ input & p & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 52.3\\% (4.0) \\\\\n",
      "resnet18 & grad. $\\times$ input & p & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 62.7\\% (1.2) \\\\\n",
      "resnet18 & grad. $\\times$ input & t & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 52.3\\% (4.1) \\\\\n",
      "resnet18 & grad. $\\times$ input & t & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 62.7\\% (1.3) \\\\\n",
      "vgg16 & grad. $\\times$ input & p & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 81.3\\% (16.2) \\\\\n",
      "vgg16 & grad. $\\times$ input & p & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 51.7\\% (16.8) \\\\\n",
      "vgg16 & grad. $\\times$ input & t & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 81.3\\% (16.2) \\\\\n",
      "vgg16 & grad. $\\times$ input & t & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 51.8\\% (16.8) \\\\\n",
      "resnet18 & integrated (b) & p & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.9\\% (0.0) \\\\\n",
      "resnet18 & integrated (b) & p & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 98.4\\% (2.4) \\\\\n",
      "resnet18 & integrated (b) & t & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.9\\% (0.0) \\\\\n",
      "resnet18 & integrated (b) & t & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 98.4\\% (2.4) \\\\\n",
      "vgg16 & integrated (b) & p & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & integrated (b) & p & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 93.6\\% (0.6) \\\\\n",
      "vgg16 & integrated (b) & t & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & integrated (b) & t & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 93.6\\% (0.7) \\\\\n",
      "resnet18 & integrated (w) & p & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.9\\% (0.0) \\\\\n",
      "resnet18 & integrated (w) & p & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 98.4\\% (1.3) \\\\\n",
      "resnet18 & integrated (w) & t & l & 99.8\\% (0.1) & 12.4\\% (1.7) & 99.9\\% (0.0) \\\\\n",
      "resnet18 & integrated (w) & t & s & 99.8\\% (0.1) & 12.4\\% (1.7) & 98.4\\% (1.3) \\\\\n",
      "vgg16 & integrated (w) & p & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & integrated (w) & p & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 98.3\\% (1.2) \\\\\n",
      "vgg16 & integrated (w) & t & l & 99.9\\% (0.0) & 11.5\\% (1.9) & 100.0\\% (0.0) \\\\\n",
      "vgg16 & integrated (w) & t & s & 99.9\\% (0.0) & 11.5\\% (1.9) & 98.3\\% (1.3) \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 299
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-13T16:18:38.819811Z",
     "start_time": "2024-09-13T16:18:38.799939Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df3 = df2[(df2['Attribution method'].str.contains(\"integrated\")) & (df2['use_y_true'] == False) & (df2['use_softmax'] == True)]\n",
    "\n",
    "table2App = df3.groupby([\"dataset\", \"data_seed\", \"train_pos_sd\", \"train_col_sd\", \"model_type\", \"attribution_method\", \"Attribution method\", \"use_y_true\", \"use_softmax\", \"test_col_sd\", \"test_pos_sd\", \"\\\\textbf{p}redicted/\\\\textbf{t}rue\",\n",
    "    \"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\"], as_index=False).agg({\n",
    "    \"overall.attribution_incorrect\": ['mean'],\n",
    "    \"attribution_incorrect.blackbg_withpix_model_accuracy\": [\"mean\", \"std\"],\n",
    "    \"attribution_incorrect.blackbg_withattrpix_model_accuracy\": [\"mean\", \"std\"],\n",
    "    \"attribution_incorrect.truebg_withblackpix_model_accuracy\": [\"mean\", \"std\"],\n",
    "    \"attribution_incorrect.truebg_withblackattrpix_model_accuracy\": [\"mean\", \"std\"]})\n",
    "\n",
    "table2App[('additive/discriminative', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.blackbg_withpix_model_accuracy\", inc_std=False), axis=1)\n",
    "table2App[('additive/attributed', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.blackbg_withattrpix_model_accuracy\", inc_std=False), axis=1)\n",
    "table2App[('subtractive/discriminative', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.truebg_withblackpix_model_accuracy\", inc_std=False), axis=1)\n",
    "table2App[('subtractive/attributed', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.truebg_withblackattrpix_model_accuracy\", inc_std=False), axis=1)\n",
    "\n",
    "table2App = table2App.sort_values([\"attribution_method\", \"model_type\"])\n",
    "\n",
    "latex = table2App.to_latex(columns=[\n",
    "    (\"Attribution method\", ''),\n",
    "    (\"model_type\", ''),\n",
    "    (\"additive/discriminative\", ''),\n",
    "    (\"additive/attributed\", ''),\n",
    "    (\"subtractive/discriminative\", ''),\n",
    "    (\"subtractive/attributed\", '')\n",
    "    ],\n",
    "    float_format='%.3f',\n",
    "    caption=\"Analysis of integrated gradients when it fails to point at the discriminative pixel. These results explore how important the discriminative pixel is compared to the attributed pixel. Because of the way the models learn, we cannot guarantee that the discriminative pixel is most important, however any reasonable attribution method should at least find a pixel that is at least as important as the discriminative pixel. These results show this is not the case as the attributed accuracies are worse than the discriminative ones (e.g. lower for additive, higher for subtractive).\", label=\"tab:importance-full\",\n",
    "    index=False\n",
    "    )\n",
    "print(latex.replace(\"model_type\", \"Model\").replace(\"resnet18_3x3\", \"resnet18\").replace(\"\\\\toprule\",\"\\\\toprule\\n&& \\multicolumn{2}{c}{Additive} & \\multicolumn{2}{c}{Subtractive} \\\\\\\\\\n\").replace(\"additive/\", \"\").replace(\"subtractive/\",\"\").replace(\"begin{table}\", \"begin{table}\\n\\\\centering\").replace(\" &  &  &  &  &  \\\\\\\\\",\"\").replace(\"%\", \"\\\\%\"))"
   ],
   "id": "fd59c5bb7d06102b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}\n",
      "\\centering\n",
      "\\caption{Analysis of integrated gradients when it fails to point at the discriminative pixel. These results explore how important the discriminative pixel is compared to the attributed pixel. Because of the way the models learn, we cannot guarantee that the discriminative pixel is most important, however any reasonable attribution method should at least find a pixel that is at least as important as the discriminative pixel. These results show this is not the case as the attributed accuracies are worse than the discriminative ones (e.g. lower for additive, higher for subtractive).}\n",
      "\\label{tab:importance-full}\n",
      "\\begin{tabular}{llllll}\n",
      "\\toprule\n",
      "&& \\multicolumn{2}{c}{Additive} & \\multicolumn{2}{c}{Subtractive} \\\\\n",
      "\n",
      "Model & Attribution method & discriminative & attributed & discriminative & attributed \\\\\n",
      "\n",
      "\\midrule\n",
      "resnet18 & integrated (b) & 100.0\\% & 62.2\\% & 60.6\\% & 64.8\\% \\\\\n",
      "vgg16 & integrated (b) & 100.0\\% & 94.1\\% & 95.2\\% & 99.3\\% \\\\\n",
      "resnet18 & integrated (w) & 100.0\\% & 93.1\\% & 87.1\\% & 92.9\\% \\\\\n",
      "vgg16 & integrated (w) & 100.0\\% & 69.8\\% & 70.0\\% & 97.3\\% \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 300
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-13T16:33:02.430644Z",
     "start_time": "2024-09-13T16:33:02.407271Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df3 = df[(df['dataset'] == \"SinglePixelDataset\") & (df['train_pos_sd'] == 0) & (df['test_pos_sd'] == 0) & (df['test_col_sd'] == 0) & (df['train_col_sd'] == 0) & (df[\"data_seed\"] == 1) & (df[\"attribution_method\"] != \"predict_importance_absgrad_absinp\")]\n",
    "\n",
    "\n",
    "table2App = df3.groupby([\"dataset\", \"data_seed\", \"train_pos_sd\", \"train_col_sd\", \"model_type\", \"attribution_method\", \"Attribution method\", \"use_y_true\", \"use_softmax\", \"test_col_sd\", \"test_pos_sd\", \"\\\\textbf{p}redicted/\\\\textbf{t}rue\",\n",
    "    \"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\"], as_index=False).agg({\n",
    "    \"overall.attribution_incorrect\": ['mean'],\n",
    "    \"attribution_incorrect.blackbg_withpix_model_accuracy\": [\"mean\", \"std\"],\n",
    "    \"attribution_incorrect.blackbg_withattrpix_model_accuracy\": [\"mean\", \"std\"],\n",
    "    \"attribution_incorrect.truebg_withblackpix_model_accuracy\": [\"mean\", \"std\"],\n",
    "    \"attribution_incorrect.truebg_withblackattrpix_model_accuracy\": [\"mean\", \"std\"]})\n",
    "\n",
    "table2App[('additive/discriminative', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.blackbg_withpix_model_accuracy\", inc_std=False), axis=1)\n",
    "table2App[('additive/attributed', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.blackbg_withattrpix_model_accuracy\", inc_std=False), axis=1)\n",
    "table2App[('subtractive/discriminative', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.truebg_withblackpix_model_accuracy\", inc_std=False), axis=1)\n",
    "table2App[('subtractive/attributed', '')] = table2App.apply(partial(format_acc, key=\"attribution_incorrect.truebg_withblackattrpix_model_accuracy\", inc_std=False), axis=1)\n",
    "\n",
    "table2App = table2App.sort_values([\"attribution_method\", \"model_type\"])\n",
    "\n",
    "latex = table2App.to_latex(columns=[\n",
    "    (\"Attribution method\", ''),\n",
    "    (\"model_type\", ''),\n",
    "    (\"\\\\textbf{p}redicted/\\\\textbf{t}rue\", ''),\n",
    "    (\"\\\\textbf{s}oftmax/\\\\textbf{l}ogit\", ''),\n",
    "    (\"additive/discriminative\", ''),\n",
    "    (\"additive/attributed\", ''),\n",
    "    (\"subtractive/discriminative\", ''),\n",
    "    (\"subtractive/attributed\", '')\n",
    "    ],\n",
    "    float_format='%.3f',\n",
    "    caption=\"Analysis of integrated gradients when it fails to point at the discriminative pixel. These results explore how important the discriminative pixel is compared to the attributed pixel. Because of the way the models learn, we cannot guarantee that the discriminative pixel is most important, however any reasonable attribution method should at least find a pixel that is at least as important as the discriminative pixel. These results show this is not the case.\", label=\"tab:importance-full\",\n",
    "    index=False\n",
    "    )\n",
    "print(latex.replace(\"model_type\", \"Model\").replace(\"resnet18_3x3\", \"resnet18\").replace(\"\\\\toprule\",\"\\\\toprule\\n&&\\\\textbf{p}redicted/&\\\\textbf{s}oftmax/& \\multicolumn{2}{c}{Additive} & \\multicolumn{2}{c}{Subtractive} \\\\\\\\\\n\").replace(\"additive/\", \"\").replace(\"subtractive/\",\"\").replace(\"begin{table}\", \"begin{table}[H]\\n\\\\centering\").replace(\"&  &  &  &  &  &  &  \\\\\\\\\",\"\").replace(\"& discriminative\",\"& discr.\").replace(\"& attributed\",\"& attrd.\").replace(\"& \\\\textbf{p}redicted/\",\"& \").replace(\"& \\\\textbf{s}oftmax/\",\"& \").replace(\"%\",\"\\\\%\"))"
   ],
   "id": "7a592272b5837aed",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}[H]\n",
      "\\centering\n",
      "\\caption{Analysis of integrated gradients when it fails to point at the discriminative pixel. These results explore how important the discriminative pixel is compared to the attributed pixel. Because of the way the models learn, we cannot guarantee that the discriminative pixel is most important, however any reasonable attribution method should at least find a pixel that is at least as important as the discriminative pixel. These results show this is not the case.}\n",
      "\\label{tab:importance-full}\n",
      "\\begin{tabular}{llllllll}\n",
      "\\toprule\n",
      "&&\\textbf{p}redicted/&\\textbf{s}oftmax/& \\multicolumn{2}{c}{Additive} & \\multicolumn{2}{c}{Subtractive} \\\\\n",
      "\n",
      "Model & Attribution method & \\textbf{t}rue & \\textbf{l}ogit & discr. & attrd. & discr. & attrd. \\\\\n",
      " \n",
      "\\midrule\n",
      "resnet18 & grad. mag. & p & l & 100.0\\% & 12.1\\% & 17.4\\% & 99.2\\% \\\\\n",
      "resnet18 & grad. mag. & p & s & 100.0\\% & 9.5\\% & 12.0\\% & 98.4\\% \\\\\n",
      "resnet18 & grad. mag. & t & l & 100.0\\% & 12.1\\% & 17.4\\% & 98.9\\% \\\\\n",
      "resnet18 & grad. mag. & t & s & 100.0\\% & 9.5\\% & 12.0\\% & 98.4\\% \\\\\n",
      "vgg16 & grad. mag. & p & l & 66.7\\% & 5.6\\% & 11.1\\% & 55.6\\% \\\\\n",
      "vgg16 & grad. mag. & p & s & 66.7\\% & 4.8\\% & 0.0\\% & 33.3\\% \\\\\n",
      "vgg16 & grad. mag. & t & l & 66.7\\% & 5.6\\% & 11.1\\% & 44.4\\% \\\\\n",
      "vgg16 & grad. mag. & t & s & 66.7\\% & 4.8\\% & 0.0\\% & 33.3\\% \\\\\n",
      "resnet18 & grad. $\\times$ input & p & l & 100.0\\% & 18.9\\% & 22.7\\% & 99.3\\% \\\\\n",
      "resnet18 & grad. $\\times$ input & p & s & 98.8\\% & 15.7\\% & 17.0\\% & 98.9\\% \\\\\n",
      "resnet18 & grad. $\\times$ input & t & l & 100.0\\% & 19.0\\% & 22.7\\% & 99.4\\% \\\\\n",
      "resnet18 & grad. $\\times$ input & t & s & 98.8\\% & 15.7\\% & 17.1\\% & 98.9\\% \\\\\n",
      "vgg16 & grad. $\\times$ input & p & l & 100.0\\% & 0.1\\% & 0.7\\% & 99.1\\% \\\\\n",
      "vgg16 & grad. $\\times$ input & p & s & 100.0\\% & 15.6\\% & 15.8\\% & 99.1\\% \\\\\n",
      "vgg16 & grad. $\\times$ input & t & l & 100.0\\% & 0.1\\% & 0.8\\% & 99.2\\% \\\\\n",
      "vgg16 & grad. $\\times$ input & t & s & 100.0\\% & 15.6\\% & 15.9\\% & 99.2\\% \\\\\n",
      "resnet18 & integrated (b) & p & l & 100.0\\% & 47.7\\% & 30.6\\% & 45.8\\% \\\\\n",
      "resnet18 & integrated (b) & p & s & 100.0\\% & 62.2\\% & 60.6\\% & 64.8\\% \\\\\n",
      "resnet18 & integrated (b) & t & l & 100.0\\% & 55.0\\% & 32.5\\% & 65.0\\% \\\\\n",
      "resnet18 & integrated (b) & t & s & 100.0\\% & 53.6\\% & 69.8\\% & 82.5\\% \\\\\n",
      "vgg16 & integrated (b) & p & l & 100.0\\% & 50.0\\% & 0.0\\% & 16.7\\% \\\\\n",
      "vgg16 & integrated (b) & p & s & 100.0\\% & 94.1\\% & 95.2\\% & 99.3\\% \\\\\n",
      "vgg16 & integrated (b) & t & l & 100.0\\% & 66.7\\% & 0.0\\% & 33.3\\% \\\\\n",
      "vgg16 & integrated (b) & t & s & 100.0\\% & 94.2\\% & 95.3\\% & 99.4\\% \\\\\n",
      "resnet18 & integrated (w) & p & l & 100.0\\% & 59.0\\% & 40.0\\% & 36.3\\% \\\\\n",
      "resnet18 & integrated (w) & p & s & 100.0\\% & 93.1\\% & 87.1\\% & 92.9\\% \\\\\n",
      "resnet18 & integrated (w) & t & l & 100.0\\% & 63.9\\% & 46.7\\% & 46.7\\% \\\\\n",
      "resnet18 & integrated (w) & t & s & 100.0\\% & 93.8\\% & 87.8\\% & 93.9\\% \\\\\n",
      "vgg16 & integrated (w) & p & l & 100.0\\% & 16.7\\% & 0.0\\% & 16.7\\% \\\\\n",
      "vgg16 & integrated (w) & p & s & 100.0\\% & 69.8\\% & 70.0\\% & 97.3\\% \\\\\n",
      "vgg16 & integrated (w) & t & l & 100.0\\% & 16.7\\% & 0.0\\% & 16.7\\% \\\\\n",
      "vgg16 & integrated (w) & t & s & 100.0\\% & 70.9\\% & 71.1\\% & 98.5\\% \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 312
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-13T16:08:09.095882Z",
     "start_time": "2024-09-13T16:08:09.094270Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "a72f14adca3ec9bc",
   "outputs": [],
   "execution_count": 279
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "76dc1e1876e64bd8"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}