notebook_replace_query_with_parameter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "----------\n",
    "# Environment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------\n",
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/dataset_mammography.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI_RADS</th>\n",
       "      <th>age</th>\n",
       "      <th>shape</th>\n",
       "      <th>margin</th>\n",
       "      <th>density</th>\n",
       "      <th>severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BI_RADS   age  shape  margin  density  severity\n",
       "0      5.0  67.0    3.0     5.0      3.0         1\n",
       "1      4.0  43.0    1.0     1.0      NaN         1\n",
       "2      5.0  58.0    4.0     5.0      3.0         1\n",
       "3      4.0  28.0    1.0     1.0      3.0         0\n",
       "4      5.0  74.0    1.0     5.0      NaN         1"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     float64\n",
       "age         float64\n",
       "shape       float64\n",
       "margin      float64\n",
       "density     float64\n",
       "severity      int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing Values benign: \n",
      "BI_RADS      1\n",
      "age          0\n",
      "shape       19\n",
      "margin      37\n",
      "density     54\n",
      "severity     0\n",
      "dtype: int64\n",
      "\n",
      "Missing Values malignant: \n",
      "BI_RADS      1\n",
      "age          5\n",
      "shape       12\n",
      "margin      11\n",
      "density     22\n",
      "severity     0\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "benign = df[df['severity'] == 0].isnull().sum()\n",
    "print('Missing Values benign: ')\n",
    "print(benign)\n",
    "print()\n",
    "malignant = df[df['severity'] == 1].isnull().sum()\n",
    "print('Missing Values malignant: ')\n",
    "print(malignant)\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------\n",
    "\n",
    "## Code without refactoring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def treatment_missing_values_original():\n",
    "    for col in df.columns:\n",
    "        if col != 'age':\n",
    "            mode_benign = df[(df['severity'] == 0)][col].mode()[0]\n",
    "            mode_malignant = df[(df['severity'] == 1)][col].mode()[0]\n",
    "            df.loc[(df[col].isnull())&(df['severity'] == 0), col] = mode_benign\n",
    "            df.loc[(df[col].isnull())&(df['severity'] == 1), col] = mode_malignant\n",
    "            \n",
    "        else:\n",
    "            mean_benign = df[(df['severity'] == 0)][col].mean()\n",
    "            mean_malignant = df[(df['severity'] == 1)][col].mean()\n",
    "            df.loc[(df[col].isnull())&(df['severity'] == 0), col] = mean_benign\n",
    "            df.loc[(df[col].isnull())&(df['severity'] == 1), col] = mean_malignant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['BI_RADS', 'age', 'shape', 'margin', 'density', 'severity'], dtype='object')"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "treatment_missing_values_original()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "------\n",
    "\n",
    "# Activite aplicate in classroom\n",
    "\n",
    "1. Write or choose code so that the refactoring can be applied.\n",
    "2. Identify the associated bad smell.\n",
    "3. Write test cases and run\n",
    "4. Apply the refactoring\n",
    "5. Run the test cases again\n",
    "6. Document each step of the presentation\n",
    "\n",
    "**Bad smell**\n",
    "\n",
    "- Long parameter list: The bad smell I found inside the code below was the long parameter list, for having long and unclear queries. This bad smell can bring difficulties or errors in the code."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "------\n",
    "\n",
    "# Procedure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 1º step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = pd.read_csv('data/dataset_mammography.csv')\n",
    "refactoreDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS      2\n",
       "age          5\n",
       "shape       31\n",
       "margin      48\n",
       "density     76\n",
       "severity     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def treatment_missing_values_refc_01():\n",
    "    cols = refactoreDF.columns\n",
    "    for col in cols:\n",
    "        if col != 'age':\n",
    "            mode_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mode()[0]\n",
    "            mode_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mode()[0]\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mode_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mode_malignant\n",
    "        else:\n",
    "            mean_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mean()\n",
    "            mean_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mean()\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mean_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mean_malignant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "treatment_missing_values_refc_01()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 2º step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = pd.read_csv('data/dataset_mammography.csv')\n",
    "refactoreDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS      2\n",
       "age          5\n",
       "shape       31\n",
       "margin      48\n",
       "density     76\n",
       "severity     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def XXXNEWtreatment_missing_values_refc_02(cols):\n",
    "    for col in cols:\n",
    "        if col != 'age':\n",
    "            mode_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mode()[0]\n",
    "            mode_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mode()[0]\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mode_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mode_malignant\n",
    "        else:\n",
    "            mean_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mean()\n",
    "            mean_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mean()\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mean_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mean_malignant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def XXXNEWtreatment_missing_values_refc_02_refactor(): \n",
    "    cols = refactoreDF.columns\n",
    "    return XXXNEWtreatment_missing_values_refc_02(cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "XXXNEWtreatment_missing_values_refc_02_refactor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 3º step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = pd.read_csv('data/dataset_mammography.csv')\n",
    "refactoreDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS      2\n",
       "age          5\n",
       "shape       31\n",
       "margin      48\n",
       "density     76\n",
       "severity     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def XXXNEWtreatment_missing_values_refc_03(cols):\n",
    "    for col in cols:\n",
    "        if col != 'age':\n",
    "            mode_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mode()[0]\n",
    "            mode_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mode()[0]\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mode_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mode_malignant\n",
    "        else:\n",
    "            mean_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mean()\n",
    "            mean_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mean()\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mean_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mean_malignant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def XXXNEWtreatment_missing_values_refc_03_refactor(): \n",
    "    return XXXNEWtreatment_missing_values_refc_03(refactoreDF.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "XXXNEWtreatment_missing_values_refc_03_refactor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 4º step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 358,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = pd.read_csv('data/dataset_mammography.csv')\n",
    "refactoreDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS      2\n",
       "age          5\n",
       "shape       31\n",
       "margin      48\n",
       "density     76\n",
       "severity     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def XXXNEWtreatment_missing_values_refc_04(cols):\n",
    "    for col in cols:\n",
    "        if col != 'age':\n",
    "            modebenign = refactoreDF[(refactoreDF['severity'] == 0)][col].mode()[0]\n",
    "            mode_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mode()[0]\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = modebenign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mode_malignant\n",
    "        else:\n",
    "            meanbenign = refactoreDF[(refactoreDF['severity'] == 0)][col].mean()\n",
    "            mean_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mean()\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = meanbenign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mean_malignant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#def XXXNEWtreatment_missing_values_refc_03_refactor(): \n",
    "#    return XXXNEWtreatment_missing_values_refc_03(refactoreDF.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "XXXNEWtreatment_missing_values_refc_04(refactoreDF.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 5º step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = pd.read_csv('data/dataset_mammography.csv')\n",
    "refactoreDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS      2\n",
       "age          5\n",
       "shape       31\n",
       "margin      48\n",
       "density     76\n",
       "severity     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def treatment_missing_values_refc_05(cols):\n",
    "    for col in cols:\n",
    "        if col != 'age':\n",
    "            mode_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mode()[0]\n",
    "            mode_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mode()[0]\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mode_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mode_malignant\n",
    "        else:\n",
    "            mean_benign = refactoreDF[(refactoreDF['severity'] == 0)][col].mean()\n",
    "            mean_malignant = refactoreDF[(refactoreDF['severity'] == 1)][col].mean()\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 0), col] = mean_benign\n",
    "            refactoreDF.loc[(refactoreDF[col].isnull())&(refactoreDF['severity'] == 1), col] = mean_malignant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "treatment_missing_values_refc_05(refactoreDF.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function Final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(961, 6)"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = pd.read_csv('data/dataset_mammography.csv')\n",
    "refactoreDF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS      2\n",
       "age          5\n",
       "shape       31\n",
       "margin      48\n",
       "density     76\n",
       "severity     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [],
   "source": [
    "def treatment_binary(\n",
    "                    dataframe: pd.DataFrame,\n",
    "                    cols: list,\n",
    "                    col_binary: str,\n",
    "                    method_func: str,\n",
    "                    ) -> pd.DataFrame:\n",
    "    df_output = dataframe.copy()\n",
    "\n",
    "    for col in cols:\n",
    "        for binary in [0,1]:\n",
    "            if method_func == 'mean':\n",
    "                rst = df_output[(df_output[col_binary] == binary)][col].mean()\n",
    "            elif method_func == 'mode':\n",
    "                rst = df_output[(df_output[col_binary] == binary)][col].mode()[0]\n",
    "            else:\n",
    "                raise Exception('Method incorrect, use mean or mode')\n",
    "            df_output.loc[(df_output[col].isnull())&(df_output[col_binary] == binary), col] = rst\n",
    "    return df_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI_RADS</th>\n",
       "      <th>age</th>\n",
       "      <th>shape</th>\n",
       "      <th>margin</th>\n",
       "      <th>density</th>\n",
       "      <th>severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.93617</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.93617</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BI_RADS   age  shape  margin  density  severity\n",
       "0      5.0  67.0    3.0     5.0  3.00000         1\n",
       "1      4.0  43.0    1.0     1.0  2.93617         1\n",
       "2      5.0  58.0    4.0     5.0  3.00000         1\n",
       "3      4.0  28.0    1.0     1.0  3.00000         0\n",
       "4      5.0  74.0    1.0     5.0  2.93617         1"
      ]
     },
     "execution_count": 372,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = treatment_binary(\n",
    "    dataframe = refactoreDF,\n",
    "    cols = ['margin', 'density', 'BI_RADS', 'shape'],\n",
    "    col_binary = 'severity',\n",
    "    method_func = 'mean',\n",
    ")\n",
    "refactoreDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI_RADS</th>\n",
       "      <th>age</th>\n",
       "      <th>shape</th>\n",
       "      <th>margin</th>\n",
       "      <th>density</th>\n",
       "      <th>severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.93617</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.93617</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BI_RADS   age  shape  margin  density  severity\n",
       "0      5.0  67.0    3.0     5.0  3.00000         1\n",
       "1      4.0  43.0    1.0     1.0  2.93617         1\n",
       "2      5.0  58.0    4.0     5.0  3.00000         1\n",
       "3      4.0  28.0    1.0     1.0  3.00000         0\n",
       "4      5.0  74.0    1.0     5.0  2.93617         1"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF = treatment_binary(\n",
    "    dataframe = refactoreDF,\n",
    "    cols = ['age'],\n",
    "    col_binary = 'severity',\n",
    "    method_func = 'mode',\n",
    ")\n",
    "\n",
    "refactoreDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI_RADS     0\n",
       "age         0\n",
       "shape       0\n",
       "margin      0\n",
       "density     0\n",
       "severity    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 374,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "refactoreDF.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exception:**  Note if you run this notebook with an error and then run the tests it will cause an error in the tests, the notebook must not have exceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "#refactoreDF = treatment_missing_values(\n",
    "#    dataframe = refactoreDF,\n",
    "#    cols = ['margin', 'density', 'BI_RADS', 'shape'],\n",
    "#    col_binary = 'severity',\n",
    "#    method_func = 'Cake',\n",
    "#)\n",
    "#refactoreDF.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}