IPIP-NEO-120.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "05be025f-1d6d-41eb-aa04-14081277fa0f",
   "metadata": {},
   "source": [
    "# IPIP-NEO-120 Scores 🌊\n",
    "\n",
    "Based on user responses we calculate scores in the Big-Five version [IPIP-NEO](https://ipip.ori.org/).\n",
    "\n",
    "Official repository version:\n",
    "\n",
    "- [https://osf.io/tbmh5](https://osf.io/tbmh5/)\n",
    "\n",
    "About dataset:\n",
    "\n",
    "- [https://osf.io/hgm7n](https://osf.io/hgm7n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "50578723-bc28-4121-a8d9-dd7f8a153ad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyreadstat\n",
    "\n",
    "import pandas as pd\n",
    "import traits as T\n",
    "\n",
    "from ipipneo import IpipNeo\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "pd.options.display.max_columns = 10\n",
    "pd.options.display.max_rows = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a9dc82e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ORIGINAL_IPIP_NEO_DATASET = \"./datasets/raw/IPIP120.por\"\n",
    "NQ_IPIP_NEO_DATASET = \"./datasets/refined/IPIP120-SCORES.csv.gz\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd16e99c",
   "metadata": {},
   "source": [
    "## 1. Original dataset in SPSS portable file format 🗂️"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6ea266a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASE</th>\n",
       "      <th>SEX</th>\n",
       "      <th>AGE</th>\n",
       "      <th>SEC</th>\n",
       "      <th>MIN</th>\n",
       "      <th>...</th>\n",
       "      <th>I116</th>\n",
       "      <th>I117</th>\n",
       "      <th>I118</th>\n",
       "      <th>I119</th>\n",
       "      <th>I120</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   CASE  SEX   AGE   SEC   MIN  ...  I116  I117  I118  I119 I120\n",
       "0   1.0  2.0  19.0   8.0  41.0  ...   2.0   4.0   1.0   5.0  4.0\n",
       "1   2.0  2.0  22.0  24.0  45.0  ...   4.0   2.0   3.0   4.0  3.0\n",
       "2   4.0  2.0  22.0   3.0  57.0  ...   2.0   4.0   2.0   5.0  4.0\n",
       "3   5.0  2.0  22.0  44.0   4.0  ...   1.0   5.0   3.0   5.0  3.0\n",
       "4   6.0  1.0  13.0  14.0   6.0  ...   2.0   4.0   4.0   3.0  5.0\n",
       "\n",
       "[5 rows x 130 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0, _ = pyreadstat.read_por(ORIGINAL_IPIP_NEO_DATASET)\n",
    "\n",
    "dflen = len(df0)\n",
    "assert dflen == 619150\n",
    "\n",
    "df0.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "705f1471",
   "metadata": {},
   "source": [
    "## 2. Values ​​with zero in items are removed ❌ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9f96179c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset original total: 619150\n",
      "Is there any missing value? False\n",
      "How many missing values: 0\n",
      "Removing invalid data from dataset: IPIP120\n",
      "New dataset total: 410376\n",
      "Number of countries: 241\n"
     ]
    }
   ],
   "source": [
    "df0_float_cols = df0.select_dtypes(include=[\"float64\"]).columns\n",
    "df0[df0_float_cols] = df0[df0_float_cols].astype(int)\n",
    "df0.columns = df0.columns.str.strip().str.lower()\n",
    "\n",
    "print(f\"Dataset original total: {dflen}\")\n",
    "print(f\"Is there any missing value? {df0.isnull().values.any()}\")\n",
    "print(f\"How many missing values: {df0.isnull().values.sum()}\")\n",
    "\n",
    "print(\"Removing invalid data from dataset: IPIP120\")\n",
    "df0 = df0[(df0.loc[:, \"i1\":\"i120\"] != 0).all(axis=1)]\n",
    "\n",
    "df0 = df0.dropna(inplace=False)\n",
    "\n",
    "print(f\"New dataset total: {len(df0)}\")\n",
    "print(f\"Number of countries: {len(df0.country.unique())}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "569ada20-9598-4469-9ebe-2f55d253cc0e",
   "metadata": {},
   "source": [
    "## 3. Calculates scores 🧮"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "14770ffc",
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_SCORE = []\n",
    "\n",
    "def compute_big5_score_120(r: pd.Series) -> bool | BaseException:\n",
    "    item = []\n",
    "\n",
    "    for x in range(1, 121):\n",
    "        item.append(\n",
    "            {\n",
    "                \"id_question\": x,\n",
    "                \"id_select\": r[f\"i{x}\"],\n",
    "                \"reverse_scored\": 0,\n",
    "            }\n",
    "        )\n",
    "\n",
    "    try:\n",
    "        calc_score = IpipNeo(question=120, test=True).compute(\n",
    "            sex=\"M\" if int(r[\"sex\"]) == 1 else \"F\",\n",
    "            age=int(r[\"age\"]),\n",
    "            answers={\"answers\": item},\n",
    "        )\n",
    "        calc_score[\"case\"] = r[\"case\"]\n",
    "\n",
    "        USER_SCORE.append(calc_score)\n",
    "    except BaseException as e:\n",
    "        raise BaseException(f\"Failed: {e}\")\n",
    "    return True\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "22a392b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         True\n",
       "1         True\n",
       "4         True\n",
       "5         True\n",
       "6         True\n",
       "          ... \n",
       "619142    True\n",
       "619143    True\n",
       "619144    True\n",
       "619147    True\n",
       "619149    True\n",
       "Length: 410376, dtype: bool"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assert len(USER_SCORE) == 0, \"should be zero\"\n",
    "df0.apply(compute_big5_score_120, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "025c953c",
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_SCORE_NORMALIZED = []\n",
    "\n",
    "for x in USER_SCORE:\n",
    "    try:\n",
    "        USER_SCORE_NORMALIZED.append(\n",
    "            T.extract_personalities(\n",
    "                idx=x[\"case\"], result=x[\"person\"][\"result\"][\"personalities\"]\n",
    "            )\n",
    "        )\n",
    "    except BaseException as e:\n",
    "        raise BaseException(f\"Failed: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b99cb0af",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.DataFrame(USER_SCORE_NORMALIZED)\n",
    "df1_cols_numerics = df1.select_dtypes(include=[\"int\", \"float\"]).columns\n",
    "\n",
    "assert len(df0) == len(df1), \"should be the same\"\n",
    "\n",
    "df2 = pd.merge(df0, df1, on=\"case\", how=\"inner\")\n",
    "assert len(df2) == len(df0), \"should be the same\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e152f2cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sec</th>\n",
       "      <th>min</th>\n",
       "      <th>...</th>\n",
       "      <th>facet_anger</th>\n",
       "      <th>facet_depression</th>\n",
       "      <th>facet_self_consciousness</th>\n",
       "      <th>facet_immoderation</th>\n",
       "      <th>facet_vulnerability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>8</td>\n",
       "      <td>41</td>\n",
       "      <td>...</td>\n",
       "      <td>11.549711</td>\n",
       "      <td>58.018551</td>\n",
       "      <td>39.725121</td>\n",
       "      <td>96.864432</td>\n",
       "      <td>11.048211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>24</td>\n",
       "      <td>45</td>\n",
       "      <td>...</td>\n",
       "      <td>89.843818</td>\n",
       "      <td>89.880570</td>\n",
       "      <td>57.486716</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>80.327511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>86.802916</td>\n",
       "      <td>29.000871</td>\n",
       "      <td>7.509259</td>\n",
       "      <td>6.890607</td>\n",
       "      <td>15.131043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>25</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>57.719898</td>\n",
       "      <td>18.108659</td>\n",
       "      <td>8.870845</td>\n",
       "      <td>44.831762</td>\n",
       "      <td>5.285188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>...</td>\n",
       "      <td>7.899414</td>\n",
       "      <td>26.475441</td>\n",
       "      <td>38.817649</td>\n",
       "      <td>62.697315</td>\n",
       "      <td>53.029251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>...</td>\n",
       "      <td>73.862903</td>\n",
       "      <td>32.943919</td>\n",
       "      <td>58.257428</td>\n",
       "      <td>84.677109</td>\n",
       "      <td>46.924899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>49.233759</td>\n",
       "      <td>11.912872</td>\n",
       "      <td>58.257428</td>\n",
       "      <td>55.597358</td>\n",
       "      <td>46.924899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>27</td>\n",
       "      <td>48</td>\n",
       "      <td>...</td>\n",
       "      <td>4.344188</td>\n",
       "      <td>6.334921</td>\n",
       "      <td>3.908683</td>\n",
       "      <td>12.104661</td>\n",
       "      <td>6.911371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>48</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>8.881105</td>\n",
       "      <td>22.370701</td>\n",
       "      <td>58.278921</td>\n",
       "      <td>12.269957</td>\n",
       "      <td>28.837680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>2.841271</td>\n",
       "      <td>99.651096</td>\n",
       "      <td>30.798323</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.048211</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 165 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   case  sex  age  sec  min  ...  facet_anger  facet_depression  \\\n",
       "0     1    2   19    8   41  ...    11.549711         58.018551   \n",
       "1     2    2   22   24   45  ...    89.843818         89.880570   \n",
       "2     6    1   13   14    6  ...    86.802916         29.000871   \n",
       "3     7    2   18   25   11  ...    57.719898         18.108659   \n",
       "4     8    2   24   19   25  ...     7.899414         26.475441   \n",
       "5     9    2   20   26   30  ...    73.862903         32.943919   \n",
       "6    10    2   18   18   36  ...    49.233759         11.912872   \n",
       "7    12    1   50   27   48  ...     4.344188          6.334921   \n",
       "8    13    1   28   48   50  ...     8.881105         22.370701   \n",
       "9    16    2   19   19   20  ...     2.841271         99.651096   \n",
       "\n",
       "   facet_self_consciousness  facet_immoderation facet_vulnerability  \n",
       "0                 39.725121           96.864432           11.048211  \n",
       "1                 57.486716            1.000000           80.327511  \n",
       "2                  7.509259            6.890607           15.131043  \n",
       "3                  8.870845           44.831762            5.285188  \n",
       "4                 38.817649           62.697315           53.029251  \n",
       "5                 58.257428           84.677109           46.924899  \n",
       "6                 58.257428           55.597358           46.924899  \n",
       "7                  3.908683           12.104661            6.911371  \n",
       "8                 58.278921           12.269957           28.837680  \n",
       "9                 30.798323            1.000000           11.048211  \n",
       "\n",
       "[10 rows x 165 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d905ff4c-11c6-4a1c-a424-612745720915",
   "metadata": {},
   "source": [
    "## 4. Save refined datasets ✅"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "83c7f116-582a-4f45-b1e1-5e7de1b2ddfd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset saved in: ./datasets/refined/IPIP120-SCORES.csv.gz\n"
     ]
    }
   ],
   "source": [
    "assert len(df2) == 410376, \"the count should be 410376!\"\n",
    "\n",
    "df2.to_csv(NQ_IPIP_NEO_DATASET, compression=\"gzip\", index=False)\n",
    "\n",
    "print(f\"Dataset saved in: {NQ_IPIP_NEO_DATASET}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}